{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.5"
    },
    "colab": {
      "name": "LCL_Transformer_HW05_ZH.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "cml77b_GpP_R",
        "gs0rD5vMpP_R",
        "SiNt9FUzpP_R",
        "AGlJU7zMpP_R",
        "OVkHLwpYpP_S",
        "VIDJl34epP_T",
        "zPsxOqoJpP_T",
        "bFaBn9gypP_T",
        "k7E4Tq9NpP_U",
        "BIJ_lVn4pP_V"
      ]
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "f857b6d1db15485096eac510203e20d4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_4a6829019582497a8a2a56359a1bc1b4",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_1354ea065af247089ab86cbbb2cf05cc",
              "IPY_MODEL_ab2069c9a95f4db3a0a170059c5f16a8"
            ]
          }
        },
        "4a6829019582497a8a2a56359a1bc1b4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1354ea065af247089ab86cbbb2cf05cc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_8cf83a4c9488420e9e7c0eabd4650f2d",
            "_dom_classes": [],
            "description": "train epoch 1: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d29f5bfb768e43fa85424722c25d0e7e"
          }
        },
        "ab2069c9a95f4db3a0a170059c5f16a8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d7537cc2b9564c43b89c04af54aaab56",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [03:17&lt;00:00,  4.53it/s, loss=6.13]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_770c3d6fd0834979a80a9f178e15d0d4"
          }
        },
        "8cf83a4c9488420e9e7c0eabd4650f2d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d29f5bfb768e43fa85424722c25d0e7e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d7537cc2b9564c43b89c04af54aaab56": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "770c3d6fd0834979a80a9f178e15d0d4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9ecdb4186dd14b37a2d87f1acaa827ad": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_6bda332921c547f680ad29c19a3819a1",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_27ba67eb61614c97a084a3b1526f5a4d",
              "IPY_MODEL_6aa9f53a019d410487e8295cdb535f46"
            ]
          }
        },
        "6bda332921c547f680ad29c19a3819a1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "27ba67eb61614c97a084a3b1526f5a4d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_cab77ce0c7d94dea8d0001e9a08f7b9e",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4adda82d74c34a95a92b61dc15f3509e"
          }
        },
        "6aa9f53a019d410487e8295cdb535f46": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_38c62ebcb3574038ac9844d9cf12d3ae",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:26&lt;00:00,  1.58s/it, valid_loss=6.38]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_de1e10dea8c240cca8eedb8c2ce52f15"
          }
        },
        "cab77ce0c7d94dea8d0001e9a08f7b9e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4adda82d74c34a95a92b61dc15f3509e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "38c62ebcb3574038ac9844d9cf12d3ae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "de1e10dea8c240cca8eedb8c2ce52f15": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "97422a5d2c104b51b78289c533aa4140": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_aecea25cc31f4712873fa958dd0b1cd8",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_2bfe08d6a9104cf6a9d3059215b0f7f4",
              "IPY_MODEL_38f76a378612411bb7fd54417739cff4"
            ]
          }
        },
        "aecea25cc31f4712873fa958dd0b1cd8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2bfe08d6a9104cf6a9d3059215b0f7f4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_5d5f2ed804234e80a606faec322d4b49",
            "_dom_classes": [],
            "description": "train epoch 2: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9c17c8976c1b409f8a2fda6422f67151"
          }
        },
        "38f76a378612411bb7fd54417739cff4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f06f0f3c4c974de890b19aa0cd07c1e6",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [03:18&lt;00:00,  4.43it/s, loss=4.8]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9509eeab126246c1bfb1252c2f2e6da1"
          }
        },
        "5d5f2ed804234e80a606faec322d4b49": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9c17c8976c1b409f8a2fda6422f67151": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f06f0f3c4c974de890b19aa0cd07c1e6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9509eeab126246c1bfb1252c2f2e6da1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5efc4463901548d0ae9f3c6e390305fb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_ef61d88357d74800abe2211213875c8e",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_e57e9bd98d7c491190b99a8e48e4f288",
              "IPY_MODEL_b39cf53022804d70a2a92b65e298699d"
            ]
          }
        },
        "ef61d88357d74800abe2211213875c8e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e57e9bd98d7c491190b99a8e48e4f288": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_b3b62d4dc6f94b539a70bed6c213ab64",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_94f00c3001b846949f138a599aa38eaa"
          }
        },
        "b39cf53022804d70a2a92b65e298699d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_585132bd1ec74b19b39bc1d56c3fc4c6",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:19&lt;00:00,  1.37it/s, valid_loss=6.01]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_56442fd8c35044419ad6898c5c45c66b"
          }
        },
        "b3b62d4dc6f94b539a70bed6c213ab64": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "94f00c3001b846949f138a599aa38eaa": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "585132bd1ec74b19b39bc1d56c3fc4c6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "56442fd8c35044419ad6898c5c45c66b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a1c8b59a443140ffaac9c55deff7fc17": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_cf7a89a621104564a1420c60b8e3967a",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_73376e7e5cf84ba8949e7644a4b7a9e3",
              "IPY_MODEL_9461c19314994ab798f5078b91475155"
            ]
          }
        },
        "cf7a89a621104564a1420c60b8e3967a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "73376e7e5cf84ba8949e7644a4b7a9e3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_dc01a3f28702451bba06c2ba2a99bcd5",
            "_dom_classes": [],
            "description": "train epoch 3: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_590fbc6632344811954b770e84093734"
          }
        },
        "9461c19314994ab798f5078b91475155": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_dc0f264da2a141cfadfd3184133ac52e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [03:17&lt;00:00,  4.14it/s, loss=4.7]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c2c4877ef5b8464b9b8af0ed813cffd9"
          }
        },
        "dc01a3f28702451bba06c2ba2a99bcd5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "590fbc6632344811954b770e84093734": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "dc0f264da2a141cfadfd3184133ac52e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c2c4877ef5b8464b9b8af0ed813cffd9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a77fdf78a3c744a9953aa32ebc56ba6e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_8c4cca19a54344ce92baf365686edf34",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_210e37cbc63941c78861350b27b5a767",
              "IPY_MODEL_06ec60c0099a4741b0a9c25cdd26752e"
            ]
          }
        },
        "8c4cca19a54344ce92baf365686edf34": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "210e37cbc63941c78861350b27b5a767": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_1527fc05da3b42f897ab722d6ec3ae94",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_761a321f0b62499fb85ed6f2e3fd28a0"
          }
        },
        "06ec60c0099a4741b0a9c25cdd26752e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_ecc833c7be1241e3a3b8c42b0a0413f8",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:23&lt;00:00,  1.08s/it, valid_loss=5.39]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6736f29d704b4fcda53aabff067e50f3"
          }
        },
        "1527fc05da3b42f897ab722d6ec3ae94": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "761a321f0b62499fb85ed6f2e3fd28a0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ecc833c7be1241e3a3b8c42b0a0413f8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6736f29d704b4fcda53aabff067e50f3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1dd956561a4949629cc483805c4d30ca": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_7540cb5f8c53476a9c95c4edbfbb4dab",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_37d8d63f5ac14651a1d23bb2de7048e8",
              "IPY_MODEL_cc4cbc495137415ab68470fbac5d0a21"
            ]
          }
        },
        "7540cb5f8c53476a9c95c4edbfbb4dab": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "37d8d63f5ac14651a1d23bb2de7048e8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_25274892efe0411ebe57d399f0f6d3b1",
            "_dom_classes": [],
            "description": "train epoch 4: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_73f74cab664c4e818a8940e6cb455139"
          }
        },
        "cc4cbc495137415ab68470fbac5d0a21": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_6cea2d3e1f584834b6ddda2047fc2081",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [03:18&lt;00:00,  4.34it/s, loss=4.48]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7ed74ada9b634f4c9a6a08359a54e309"
          }
        },
        "25274892efe0411ebe57d399f0f6d3b1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "73f74cab664c4e818a8940e6cb455139": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6cea2d3e1f584834b6ddda2047fc2081": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7ed74ada9b634f4c9a6a08359a54e309": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7a3fa24175464695a5d8bc5144fcc5d8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_664ad871c2d64964aa3a5f7c38e523e8",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_d12d47d4d90d46d194ea2ca282a16ee0",
              "IPY_MODEL_a965289cf7d9449cb982467cf6d35c87"
            ]
          }
        },
        "664ad871c2d64964aa3a5f7c38e523e8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d12d47d4d90d46d194ea2ca282a16ee0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_eb746a8bce534f4eb8b1eafce1c8e42c",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_61db40e9e5b74cdbbbbde6292b5e1009"
          }
        },
        "a965289cf7d9449cb982467cf6d35c87": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_8e7dd65ae690444c9ab5afbf938159a2",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:24&lt;00:00,  1.23s/it, valid_loss=5.34]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_06f51b1647d04b98ac1505b9d9541a84"
          }
        },
        "eb746a8bce534f4eb8b1eafce1c8e42c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "61db40e9e5b74cdbbbbde6292b5e1009": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8e7dd65ae690444c9ab5afbf938159a2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "06f51b1647d04b98ac1505b9d9541a84": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9347496d609b4d22a7b13ecf0bb44320": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_30a2db8f759147a18cc1523123183acd",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_877616819dfa4a2185230dcd5b4b393a",
              "IPY_MODEL_44f37691eaba41c488d99c2fe805bad6"
            ]
          }
        },
        "30a2db8f759147a18cc1523123183acd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "877616819dfa4a2185230dcd5b4b393a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_5611d68808e9408f9e668d620377331a",
            "_dom_classes": [],
            "description": "train epoch 5: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_dccad50d188141178b2e18ccb8538a43"
          }
        },
        "44f37691eaba41c488d99c2fe805bad6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_869ab53cc26943599d01be2cbc80e937",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [03:17&lt;00:00,  4.17it/s, loss=4.3]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4186d8f12a22414f88e7167988955bdb"
          }
        },
        "5611d68808e9408f9e668d620377331a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "dccad50d188141178b2e18ccb8538a43": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "869ab53cc26943599d01be2cbc80e937": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4186d8f12a22414f88e7167988955bdb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ece3c14c188243baac9d1682c8005859": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_3a27d8a9408243faae99dc7ad12e7d33",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_19e99c5edaa4472487c43cd879cd3c72",
              "IPY_MODEL_3b0bf4a83d2548a687adf3f10b2b122b"
            ]
          }
        },
        "3a27d8a9408243faae99dc7ad12e7d33": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "19e99c5edaa4472487c43cd879cd3c72": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_b7dd9f38300d4fdaa11bbf499b4572a9",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_13f6ac24e3524b0ba037b20467b9c5fe"
          }
        },
        "3b0bf4a83d2548a687adf3f10b2b122b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d645b28b4b2342efb3d739f2fd2b6709",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:24&lt;00:00,  1.20s/it, valid_loss=5.16]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_fff9be908d0742b1a58d85fc462d28b9"
          }
        },
        "b7dd9f38300d4fdaa11bbf499b4572a9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "13f6ac24e3524b0ba037b20467b9c5fe": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d645b28b4b2342efb3d739f2fd2b6709": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "fff9be908d0742b1a58d85fc462d28b9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e3898759a14d4e21aea94e421a4bcd29": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_c2b648403f8e45b1be1869958e40f3ae",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_cd61247b7d394b5da7210bae1acf4e8e",
              "IPY_MODEL_d4f93415846e471691b6a4bdbd09bf11"
            ]
          }
        },
        "c2b648403f8e45b1be1869958e40f3ae": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "cd61247b7d394b5da7210bae1acf4e8e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_1b160eb55a5c4e43bad219130fd43dc9",
            "_dom_classes": [],
            "description": "train epoch 6: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_099344472f1d422f8f93e7d05c308e2e"
          }
        },
        "d4f93415846e471691b6a4bdbd09bf11": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_324ede7d621f45d397bdb41970360b43",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [03:18&lt;00:00,  4.21it/s, loss=3.87]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_411006be765b4aa4ada028f055639080"
          }
        },
        "1b160eb55a5c4e43bad219130fd43dc9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "099344472f1d422f8f93e7d05c308e2e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "324ede7d621f45d397bdb41970360b43": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "411006be765b4aa4ada028f055639080": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "afd568333e1641faa1ad279c13d1c630": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_3aeac31a2fdd449382145a7702a0107e",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_b455e18a90974fb7a6f0169ca880b9e8",
              "IPY_MODEL_c6513dd1e6bb46ba9661fc120228b3ce"
            ]
          }
        },
        "3aeac31a2fdd449382145a7702a0107e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b455e18a90974fb7a6f0169ca880b9e8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_e36a560e62e147f0a5ba17815a6dd385",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d01c0a37e0dd4f16893d5e6a5f60a162"
          }
        },
        "c6513dd1e6bb46ba9661fc120228b3ce": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_e48264c1f23d4feaa7802f35c101da03",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:24&lt;00:00,  1.51s/it, valid_loss=5.24]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_fb03f85f832e4dceab9c414dc17ac47e"
          }
        },
        "e36a560e62e147f0a5ba17815a6dd385": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d01c0a37e0dd4f16893d5e6a5f60a162": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e48264c1f23d4feaa7802f35c101da03": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "fb03f85f832e4dceab9c414dc17ac47e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "864094c044844e0aa70aa6ee04d14a73": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_9cbce7e0ddea46f6aa9a58ee226608a6",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_f6b82034f5b444c6badb80045fd4d25f",
              "IPY_MODEL_56b6a058ebdd47adb0d0de1b95694f56"
            ]
          }
        },
        "9cbce7e0ddea46f6aa9a58ee226608a6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f6b82034f5b444c6badb80045fd4d25f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_c071f45623f94bfa8404097847f73aa5",
            "_dom_classes": [],
            "description": "train epoch 7: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4c4afdcd754f43ed95a604a4945f4828"
          }
        },
        "56b6a058ebdd47adb0d0de1b95694f56": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_11c9dad426ff46c7beeefbcfecdabf11",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [03:17&lt;00:00,  4.43it/s, loss=4.21]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ed33815ca4d84ffd9820132258ecb3a5"
          }
        },
        "c071f45623f94bfa8404097847f73aa5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4c4afdcd754f43ed95a604a4945f4828": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "11c9dad426ff46c7beeefbcfecdabf11": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ed33815ca4d84ffd9820132258ecb3a5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8d2daceaa80041629ad96f67f50ec2d7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_723ced0add5c4612a73479a5a8f2eaf1",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_3d8c2b727da3450ab552aa17dbbd08fb",
              "IPY_MODEL_09eeb1f462cc40bdabc46ed73710cf62"
            ]
          }
        },
        "723ced0add5c4612a73479a5a8f2eaf1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3d8c2b727da3450ab552aa17dbbd08fb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_ae19af82b04f437db4f489842936fb70",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6f966090fdad4233b9230d053ed5e49b"
          }
        },
        "09eeb1f462cc40bdabc46ed73710cf62": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_135b5722644849c5829f1b51a69c9734",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:23&lt;00:00,  1.42s/it, valid_loss=5.08]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_248bb2ccbfe14f5984aa6eab1fd1c698"
          }
        },
        "ae19af82b04f437db4f489842936fb70": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6f966090fdad4233b9230d053ed5e49b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "135b5722644849c5829f1b51a69c9734": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "248bb2ccbfe14f5984aa6eab1fd1c698": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1e808665cdfa4a5998f26c3d4b51c7a5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_3f4261ad57314809bc0c8b48f9975325",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_729c0b469e774502abfd47d6dbf73167",
              "IPY_MODEL_5fd99d62217342ec9815b4a1b9b40e5a"
            ]
          }
        },
        "3f4261ad57314809bc0c8b48f9975325": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "729c0b469e774502abfd47d6dbf73167": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a47ac6f3f1054ad7abb336f583105469",
            "_dom_classes": [],
            "description": "train epoch 8: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_61600384caf64d2297c4c38279a72bb1"
          }
        },
        "5fd99d62217342ec9815b4a1b9b40e5a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_c95435ed0843475e8345e3e725a7e24c",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [03:18&lt;00:00,  4.46it/s, loss=4.11]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7e1dce0e22354f7f8fd285a4c3988427"
          }
        },
        "a47ac6f3f1054ad7abb336f583105469": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "61600384caf64d2297c4c38279a72bb1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c95435ed0843475e8345e3e725a7e24c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7e1dce0e22354f7f8fd285a4c3988427": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "da42ef922b904b76811fa04efbea3ba9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_9cc9be5dbd1c457982023de386170d58",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_6338507297a443dabd413a97799b45f7",
              "IPY_MODEL_b95e56348f3042ef95b1643d5ff2d64d"
            ]
          }
        },
        "9cc9be5dbd1c457982023de386170d58": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6338507297a443dabd413a97799b45f7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_e0a7cd2190b64253868e09796c366a22",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4bca9d40b7c149f1a5ab3ce6c32bab67"
          }
        },
        "b95e56348f3042ef95b1643d5ff2d64d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_b23655f2d9aa46e7906445f6dd287a22",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:23&lt;00:00,  1.16s/it, valid_loss=4.92]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_52b8448b765b45b28eda7083a492dedd"
          }
        },
        "e0a7cd2190b64253868e09796c366a22": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4bca9d40b7c149f1a5ab3ce6c32bab67": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b23655f2d9aa46e7906445f6dd287a22": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "52b8448b765b45b28eda7083a492dedd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3490001381064d6b94e096891c8b656d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_17b7d18d68a94d36abd10fd4b1d5a420",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_2299165bf4b0431ba4e2c1d71cc4d655",
              "IPY_MODEL_73bc525624c14f978a2867a4da18c24a"
            ]
          }
        },
        "17b7d18d68a94d36abd10fd4b1d5a420": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2299165bf4b0431ba4e2c1d71cc4d655": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_707e0846832b4b88b807b46edc3bf8be",
            "_dom_classes": [],
            "description": "train epoch 9: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_cfcb419a0705446ebde311bcc6024cd8"
          }
        },
        "73bc525624c14f978a2867a4da18c24a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_094013d7f9d448e39bc0034c943a76e8",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [03:17&lt;00:00,  3.96it/s, loss=3.51]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e6e13878f94143669b6826511cb74db3"
          }
        },
        "707e0846832b4b88b807b46edc3bf8be": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "cfcb419a0705446ebde311bcc6024cd8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "094013d7f9d448e39bc0034c943a76e8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e6e13878f94143669b6826511cb74db3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ef66637c0f884002b233737b7e458feb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_13823c80b8c34f90b01dc8e53de1ed31",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_b0fa20e522514c53a0a6d8140a948f15",
              "IPY_MODEL_36e08f2f0b1f425fa950c2579f82d9ff"
            ]
          }
        },
        "13823c80b8c34f90b01dc8e53de1ed31": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b0fa20e522514c53a0a6d8140a948f15": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_6bbd28d29b244995a37da0e0636d24c6",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_478062b7c07444f19686bdd9a3ca40a9"
          }
        },
        "36e08f2f0b1f425fa950c2579f82d9ff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_c82263539410440cada74d2b50afc0ea",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:24&lt;00:00,  1.21s/it, valid_loss=4.94]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9e42e6ad779641678b2e77f126fa162d"
          }
        },
        "6bbd28d29b244995a37da0e0636d24c6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "478062b7c07444f19686bdd9a3ca40a9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c82263539410440cada74d2b50afc0ea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9e42e6ad779641678b2e77f126fa162d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d8ee47410b0a4556a0163485c26a1429": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_f707df215a0142edbba2b8842b698599",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_a7f4ebb409e5431981d3b7dac979931a",
              "IPY_MODEL_21f7b29bf2814f209280d34b1a8d07a7"
            ]
          }
        },
        "f707df215a0142edbba2b8842b698599": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a7f4ebb409e5431981d3b7dac979931a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_105ddf9aaa3549bba1e314f3f8266a84",
            "_dom_classes": [],
            "description": "train epoch 10: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_de8f58d2112447e18fb98ed2a690028c"
          }
        },
        "21f7b29bf2814f209280d34b1a8d07a7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_9235581a300c4dc1aeca1c650669ef52",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [03:17&lt;00:00,  4.31it/s, loss=3.88]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d101a5702e6e4860a39b968fe75ee76d"
          }
        },
        "105ddf9aaa3549bba1e314f3f8266a84": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "de8f58d2112447e18fb98ed2a690028c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9235581a300c4dc1aeca1c650669ef52": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d101a5702e6e4860a39b968fe75ee76d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "582a89642fa64656b63071f49b22796d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_a98a39f0ee6e467882809c23f0397d81",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_e49d63f1446d4439ac439f9e3f86d684",
              "IPY_MODEL_51d6496f7af349508b3a383c48bf65a0"
            ]
          }
        },
        "a98a39f0ee6e467882809c23f0397d81": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e49d63f1446d4439ac439f9e3f86d684": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_485c28bd5ff9423e9236de81cb7324df",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_31a03080aa0242c9b69b6f95036fb291"
          }
        },
        "51d6496f7af349508b3a383c48bf65a0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_1d3f98937ea94d609ad800d44450eb84",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:21&lt;00:00,  1.00s/it, valid_loss=4.88]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9c112d80f73e45aea6bee9bb67d60613"
          }
        },
        "485c28bd5ff9423e9236de81cb7324df": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "31a03080aa0242c9b69b6f95036fb291": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1d3f98937ea94d609ad800d44450eb84": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9c112d80f73e45aea6bee9bb67d60613": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fafc50e23a88426182fd07a779935b31": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_b0830c9b0d9e4d9fa0f9558c15d5a11f",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_b07e5bb52b3f4a1e8a8a43372fab881c",
              "IPY_MODEL_205206e946004a96afe7833dac832034"
            ]
          }
        },
        "b0830c9b0d9e4d9fa0f9558c15d5a11f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b07e5bb52b3f4a1e8a8a43372fab881c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a2cd48d861474c0aa1872356949e046b",
            "_dom_classes": [],
            "description": "train epoch 11: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_bf6cacaeb81549aeb296d5262214587f"
          }
        },
        "205206e946004a96afe7833dac832034": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_41e1c6bef2cb4164ac87e04b05084e88",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [03:18&lt;00:00,  4.29it/s, loss=3.76]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3ba8a17a4cbe42388c23e44bef6ff64e"
          }
        },
        "a2cd48d861474c0aa1872356949e046b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "bf6cacaeb81549aeb296d5262214587f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "41e1c6bef2cb4164ac87e04b05084e88": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3ba8a17a4cbe42388c23e44bef6ff64e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b1794d354a864a7aabc11c5d2254fa13": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_e63605bcc5f744e691bca858b4cedb3b",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_5bfb1d2e0d484d4faea590975d73e282",
              "IPY_MODEL_1885706b279d4d3c93f5892bcc1b850c"
            ]
          }
        },
        "e63605bcc5f744e691bca858b4cedb3b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5bfb1d2e0d484d4faea590975d73e282": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_be77f324e1c74fdd8e1ec101de1f4ab6",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_522e647e4cbe40af9f3c7b0bf8024bf6"
          }
        },
        "1885706b279d4d3c93f5892bcc1b850c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_600377cc97ed4963ac8974554c59ab91",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:22&lt;00:00,  1.04it/s, valid_loss=4.86]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b82be97fb1cb41878c237d047e95b334"
          }
        },
        "be77f324e1c74fdd8e1ec101de1f4ab6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "522e647e4cbe40af9f3c7b0bf8024bf6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "600377cc97ed4963ac8974554c59ab91": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b82be97fb1cb41878c237d047e95b334": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3d3da426d14f4da098753388612a6f4c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_f7f9c2583da047239f090e3d0c359195",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_a33210c7a85a45129b41ed523ba4d288",
              "IPY_MODEL_c70e099e775e4d84b2ebb58fde27a574"
            ]
          }
        },
        "f7f9c2583da047239f090e3d0c359195": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a33210c7a85a45129b41ed523ba4d288": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_176267925e1b4f60a135e64250029b04",
            "_dom_classes": [],
            "description": "train epoch 12: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4d7ca84f014c41deb60285969de910f5"
          }
        },
        "c70e099e775e4d84b2ebb58fde27a574": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_74f2649ea45749809be2c4498d92952d",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [03:17&lt;00:00,  4.45it/s, loss=3.57]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3bf0f082e18344c8a58ff0f74af1d3a5"
          }
        },
        "176267925e1b4f60a135e64250029b04": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4d7ca84f014c41deb60285969de910f5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "74f2649ea45749809be2c4498d92952d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3bf0f082e18344c8a58ff0f74af1d3a5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d23b4df531884201911c24998813416c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_5c4db674f0c0412fa37b41aba88df130",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_27043264e83d43ef804105614e24c6db",
              "IPY_MODEL_32d647faa2cb4034939b9e7085515046"
            ]
          }
        },
        "5c4db674f0c0412fa37b41aba88df130": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "27043264e83d43ef804105614e24c6db": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_50827286682946c8860b9e45f9621f64",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_799670136ce140a68835e6f548b255f9"
          }
        },
        "32d647faa2cb4034939b9e7085515046": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_41b9cd1fdb63436fb4d1cf6c5f1434d4",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:21&lt;00:00,  1.03it/s, valid_loss=4.75]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7639fb2acc764e918a73291206db55b0"
          }
        },
        "50827286682946c8860b9e45f9621f64": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "799670136ce140a68835e6f548b255f9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "41b9cd1fdb63436fb4d1cf6c5f1434d4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7639fb2acc764e918a73291206db55b0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "056fc96039164220854a7e0e9e8c2c0e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_a08cfe88e4ae45378a8a993292e73232",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_ce9c5e2d3bbe4ed9a02671bbc322e626",
              "IPY_MODEL_f331275dd6fc4719a9f75bee084c7bdd"
            ]
          }
        },
        "a08cfe88e4ae45378a8a993292e73232": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ce9c5e2d3bbe4ed9a02671bbc322e626": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_2ad3288b4bf040aca232d6a9530750dc",
            "_dom_classes": [],
            "description": "train epoch 13: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_34a6a01c2b914d209df92429b60f2f8f"
          }
        },
        "f331275dd6fc4719a9f75bee084c7bdd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_adf383f9d1274e409849405b1d73ce51",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [03:16&lt;00:00,  4.41it/s, loss=3.82]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ae02c0c0fd304e079528b9b255184790"
          }
        },
        "2ad3288b4bf040aca232d6a9530750dc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "34a6a01c2b914d209df92429b60f2f8f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "adf383f9d1274e409849405b1d73ce51": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ae02c0c0fd304e079528b9b255184790": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7ba282ed8b234aa5835fc0f46b0cc9b4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_aca9412a528743ff9a0459ce9f258e72",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_cfe9820e6e464d6180d0876508d2cb7b",
              "IPY_MODEL_ae8bfc940dab40fcb714b3ca239a9eb6"
            ]
          }
        },
        "aca9412a528743ff9a0459ce9f258e72": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "cfe9820e6e464d6180d0876508d2cb7b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_db3d88e3a5da41feb05a0e1ec89aefe5",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_99e3e37be7304cb0b03ee8d0f268d544"
          }
        },
        "ae8bfc940dab40fcb714b3ca239a9eb6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_50a35b5286874cc8ae0e919c2690f0d4",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:22&lt;00:00,  1.35s/it, valid_loss=4.71]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_43786826fff2466e9a2a18439a5a0544"
          }
        },
        "db3d88e3a5da41feb05a0e1ec89aefe5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "99e3e37be7304cb0b03ee8d0f268d544": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "50a35b5286874cc8ae0e919c2690f0d4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "43786826fff2466e9a2a18439a5a0544": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "db1f4908985f428bb8d5413f876f6d93": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_8f62a6bab64b4bc2bae81230dc7bdb6b",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_541c657fc4a04a258b5a9300973a67b4",
              "IPY_MODEL_1eb61e2aa5444b79b9e800fed7e0e252"
            ]
          }
        },
        "8f62a6bab64b4bc2bae81230dc7bdb6b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "541c657fc4a04a258b5a9300973a67b4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_e507043b9ff344c180996a771b918af4",
            "_dom_classes": [],
            "description": "train epoch 14: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_108084cd0c0b488f82f25f6b6734315c"
          }
        },
        "1eb61e2aa5444b79b9e800fed7e0e252": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_699fd30fa0f14ac887818f4dbbc21af8",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [03:18&lt;00:00,  4.44it/s, loss=4.03]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3393eb56a31b404f8af7ca5db405d367"
          }
        },
        "e507043b9ff344c180996a771b918af4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "108084cd0c0b488f82f25f6b6734315c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "699fd30fa0f14ac887818f4dbbc21af8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3393eb56a31b404f8af7ca5db405d367": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f0ba7e145d0d4971be09662f2e2fbf3a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_4334aff5f56f45238b30de33aa9157ae",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_98605c2b74f84fd797763af808cd408b",
              "IPY_MODEL_c1a89341e52c49b8b6a2b50219ec5bb7"
            ]
          }
        },
        "4334aff5f56f45238b30de33aa9157ae": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "98605c2b74f84fd797763af808cd408b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_b21049cf1a284d6a85cf5dfe0c3374f5",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b5f6875abb2e42b39034222df9d6fa3b"
          }
        },
        "c1a89341e52c49b8b6a2b50219ec5bb7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_6c4bf83ed932435e8f2760b931b098ee",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:20&lt;00:00,  1.12it/s, valid_loss=4.83]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6716b40c827b462385c72805dc6c130a"
          }
        },
        "b21049cf1a284d6a85cf5dfe0c3374f5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b5f6875abb2e42b39034222df9d6fa3b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6c4bf83ed932435e8f2760b931b098ee": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6716b40c827b462385c72805dc6c130a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "523d21c12e8144fe85d2bc5d8b8b87de": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_e9368cfc54c848d9bd337845b2b19f61",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_b25dc878560c40d281f9086b90fa2d0c",
              "IPY_MODEL_d8ae2c801c9d4720b284f5b5d600a123"
            ]
          }
        },
        "e9368cfc54c848d9bd337845b2b19f61": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b25dc878560c40d281f9086b90fa2d0c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_745b288a158d4459a52bc44e60740079",
            "_dom_classes": [],
            "description": "train epoch 15: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_58d1aa331fa746308db890ea03f11dc7"
          }
        },
        "d8ae2c801c9d4720b284f5b5d600a123": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_0be3265c7cfc47a48ada725efbc96417",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [03:19&lt;00:00,  4.55it/s, loss=3.66]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_31d3edd7d9f94439adafd80473a9a85e"
          }
        },
        "745b288a158d4459a52bc44e60740079": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "58d1aa331fa746308db890ea03f11dc7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0be3265c7cfc47a48ada725efbc96417": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "31d3edd7d9f94439adafd80473a9a85e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "cf92a0bfd2654884aaaf30dc7941aaf0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_c396333375de494e83fc7c2e70a97dd1",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_79440766449a497c9c7d3684b9a70996",
              "IPY_MODEL_249ce3d647604fdb842d2975a6addfaa"
            ]
          }
        },
        "c396333375de494e83fc7c2e70a97dd1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "79440766449a497c9c7d3684b9a70996": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_2709d8ff97e847dcb788d497ea7688f8",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b6a3415e85124a2f88d2b0e250b40be1"
          }
        },
        "249ce3d647604fdb842d2975a6addfaa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_4fd6c7dc428e4d53a23a300b89918f3c",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:21&lt;00:00,  1.08it/s, valid_loss=4.74]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_05b7528ddc1d4c0d9b90690a9fc8a383"
          }
        },
        "2709d8ff97e847dcb788d497ea7688f8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b6a3415e85124a2f88d2b0e250b40be1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4fd6c7dc428e4d53a23a300b89918f3c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "05b7528ddc1d4c0d9b90690a9fc8a383": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "aa8b0134139a4514bd938aa40df3314a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_48f34dba9634479b878d75a6859d9fa5",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_08527c2450fb45b48fecfe2179166a1d",
              "IPY_MODEL_31ee05b34eb04527ac0393a84db9ce64"
            ]
          }
        },
        "48f34dba9634479b878d75a6859d9fa5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "08527c2450fb45b48fecfe2179166a1d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_184cb124eeab463a99192375b7b1c3d1",
            "_dom_classes": [],
            "description": "train epoch 16: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c44fb1c1e2cf44dcb1107dbdfd32fdd5"
          }
        },
        "31ee05b34eb04527ac0393a84db9ce64": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_091a0d1bd10e4492b3bf2ddb03edfb70",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [03:18&lt;00:00,  4.42it/s, loss=3.47]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_50ddf8942dde4614b031e75b35d47924"
          }
        },
        "184cb124eeab463a99192375b7b1c3d1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c44fb1c1e2cf44dcb1107dbdfd32fdd5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "091a0d1bd10e4492b3bf2ddb03edfb70": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "50ddf8942dde4614b031e75b35d47924": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "daeb3902e30044cdb74f8497d5287c31": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_f8fce735712b4d83a07e199648b70362",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_756820c04d4240a7a64eae01f927e574",
              "IPY_MODEL_c18e780f7b414f01a347c3fbfc31e930"
            ]
          }
        },
        "f8fce735712b4d83a07e199648b70362": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "756820c04d4240a7a64eae01f927e574": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_93954e14055e4f6a97765bbe8417ac8a",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b8ea272386504dff947534417769bccc"
          }
        },
        "c18e780f7b414f01a347c3fbfc31e930": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_3fa3b12eee074d72abcb398e5a771e45",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:21&lt;00:00,  1.01it/s, valid_loss=4.79]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_236d5387f70f4f81982b1e1edf952a82"
          }
        },
        "93954e14055e4f6a97765bbe8417ac8a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b8ea272386504dff947534417769bccc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3fa3b12eee074d72abcb398e5a771e45": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "236d5387f70f4f81982b1e1edf952a82": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8453f3413a55457f9ff114d37bceee39": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_28f04de6c0d44e22852678ddd8ba474b",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_f3bb385458f54dbcbb2db9c1a205c779",
              "IPY_MODEL_167ef8c98d7742d09c60253e3ecb2dfe"
            ]
          }
        },
        "28f04de6c0d44e22852678ddd8ba474b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f3bb385458f54dbcbb2db9c1a205c779": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_cdd8889aaacb487e809734482bacbe04",
            "_dom_classes": [],
            "description": "train epoch 17: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7300647e149c4edf801b190e29fb52e0"
          }
        },
        "167ef8c98d7742d09c60253e3ecb2dfe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_1e4b27ed3dc948169c272bfb201a6961",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [03:19&lt;00:00,  4.45it/s, loss=3.62]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_654ff33ec18046489c3ccb48f4ef628b"
          }
        },
        "cdd8889aaacb487e809734482bacbe04": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7300647e149c4edf801b190e29fb52e0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1e4b27ed3dc948169c272bfb201a6961": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "654ff33ec18046489c3ccb48f4ef628b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e6e392d13e9545bf987a61139d5445b6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_736bf193324f45bb89f154b7d2510ab0",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_aecbb89d2268479fba0b126b1b93c19d",
              "IPY_MODEL_ff6166e0d1bd4722b4007a9a3e7eaa31"
            ]
          }
        },
        "736bf193324f45bb89f154b7d2510ab0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "aecbb89d2268479fba0b126b1b93c19d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_f63ab431f4a74a109b6bf1fdf12cb9f6",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8c31ce1324fd484dbd52981a5ee426b6"
          }
        },
        "ff6166e0d1bd4722b4007a9a3e7eaa31": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_00ec4615653d4abaa02e449063998014",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:21&lt;00:00,  1.06s/it, valid_loss=4.77]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_cdc3547313644b65aea412624585926f"
          }
        },
        "f63ab431f4a74a109b6bf1fdf12cb9f6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8c31ce1324fd484dbd52981a5ee426b6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "00ec4615653d4abaa02e449063998014": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "cdc3547313644b65aea412624585926f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6022dba9c0fc44928150f03e4d0707f8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_c998d4f29cc34995860f46282c7b7138",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_b1d6037240cd4696ac04c11070574544",
              "IPY_MODEL_dc6f67cf20944a568126e44c65656164"
            ]
          }
        },
        "c998d4f29cc34995860f46282c7b7138": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b1d6037240cd4696ac04c11070574544": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_99a6e7782a7c468594583dc52285b8ab",
            "_dom_classes": [],
            "description": "train epoch 18: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3a13dd610a584ea1bc9de59411e8459f"
          }
        },
        "dc6f67cf20944a568126e44c65656164": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_1d8ec79053314cd48730f6ed43b76d96",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [03:20&lt;00:00,  3.98it/s, loss=3.4]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5f742d7673954b628f7d2ceb809b2ab4"
          }
        },
        "99a6e7782a7c468594583dc52285b8ab": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3a13dd610a584ea1bc9de59411e8459f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1d8ec79053314cd48730f6ed43b76d96": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5f742d7673954b628f7d2ceb809b2ab4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1046bb04ed3540ffad21248155f92a59": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_c4cc9cc069924334b86d3b8d3f3f6cad",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_179f7743beac48e4b72ed2446b02d77b",
              "IPY_MODEL_02d41d1e5f254064bf2da346629f6ed4"
            ]
          }
        },
        "c4cc9cc069924334b86d3b8d3f3f6cad": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "179f7743beac48e4b72ed2446b02d77b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_c64f8b6f3dec4bab9c970f9db13b4a1d",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c0780f1d5d3e469e88b7444f4e3e4278"
          }
        },
        "02d41d1e5f254064bf2da346629f6ed4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_fbdb59a8f879435f804bd60b394959b2",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:21&lt;00:00,  1.03it/s, valid_loss=4.79]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ac9f1d823fcc4e6696605c4c29e065da"
          }
        },
        "c64f8b6f3dec4bab9c970f9db13b4a1d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c0780f1d5d3e469e88b7444f4e3e4278": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fbdb59a8f879435f804bd60b394959b2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ac9f1d823fcc4e6696605c4c29e065da": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "30bcbe1ee7f84a8ca8f9d40f77be522b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_dd14cb45f3594a0a9b7e0f14f9515e58",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_8330a5ba7a3b4ae9a6a134d72f36b8da",
              "IPY_MODEL_a67268ea3bfb41f0b0b5946399fc18ac"
            ]
          }
        },
        "dd14cb45f3594a0a9b7e0f14f9515e58": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8330a5ba7a3b4ae9a6a134d72f36b8da": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_2f2f7306afba4cc3a18ac26cb83d1e78",
            "_dom_classes": [],
            "description": "train epoch 19: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_61a5c0e56e8a4e7098f519b15a5f84e6"
          }
        },
        "a67268ea3bfb41f0b0b5946399fc18ac": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_3897f59d5536406e9c670c5be54a9867",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [03:20&lt;00:00,  4.12it/s, loss=3.37]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9925fe9372be4080b5122d5b47f38d78"
          }
        },
        "2f2f7306afba4cc3a18ac26cb83d1e78": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "61a5c0e56e8a4e7098f519b15a5f84e6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3897f59d5536406e9c670c5be54a9867": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9925fe9372be4080b5122d5b47f38d78": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "13eba79cc9f04134801a52f71881e8b1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_bd1c4ab320be4be6946ea73a74328837",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_1651e4fce18046bbb2cfd7a86dde77ad",
              "IPY_MODEL_5870ac198e4d4d7bb6e1d6f126a07266"
            ]
          }
        },
        "bd1c4ab320be4be6946ea73a74328837": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1651e4fce18046bbb2cfd7a86dde77ad": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_e475847bc2fa4954a1fae4b644da3b6d",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a8f6f7330fc04ac4a67bf2e68ba2ccb1"
          }
        },
        "5870ac198e4d4d7bb6e1d6f126a07266": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_c6f6a0f898ce411686cf966e3644fe06",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:21&lt;00:00,  1.03it/s, valid_loss=4.81]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_83133b3cd0e748208dfbecdff338af83"
          }
        },
        "e475847bc2fa4954a1fae4b644da3b6d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a8f6f7330fc04ac4a67bf2e68ba2ccb1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c6f6a0f898ce411686cf966e3644fe06": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "83133b3cd0e748208dfbecdff338af83": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c7bca190a59d44d0b41399002a2accae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_367a68bb6f034d8fbfc72228cc6e6f69",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_b4c810084d4d4eb98c49118730578356",
              "IPY_MODEL_ffd1ef53347447858f3e1078b5c1b1bb"
            ]
          }
        },
        "367a68bb6f034d8fbfc72228cc6e6f69": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b4c810084d4d4eb98c49118730578356": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_111c42d590054d519ccda957bd6abe19",
            "_dom_classes": [],
            "description": "train epoch 20: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d4168759ae6541899aa472d63c5d7e61"
          }
        },
        "ffd1ef53347447858f3e1078b5c1b1bb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_e20d80173a08410c8c67e96dc4c84e17",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [03:21&lt;00:00,  4.46it/s, loss=3.42]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_aebc6fddfd9b4b5b8aca75d9b85cad10"
          }
        },
        "111c42d590054d519ccda957bd6abe19": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d4168759ae6541899aa472d63c5d7e61": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e20d80173a08410c8c67e96dc4c84e17": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "aebc6fddfd9b4b5b8aca75d9b85cad10": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "df8ad6dae6424015baef44d5e019d13d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_45a944d3204a4bb8a724d1e12046e2c2",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_f3e1c80ac9c0468fa0f77e16b711153a",
              "IPY_MODEL_f967aec337694a94b8b45bfd6358ed28"
            ]
          }
        },
        "45a944d3204a4bb8a724d1e12046e2c2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f3e1c80ac9c0468fa0f77e16b711153a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_bde4bb315d6041cea08fca0e255a2f11",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b656d24293ee4964900d1126056780e8"
          }
        },
        "f967aec337694a94b8b45bfd6358ed28": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_88a3f3e62b2e41df87b8105688801b98",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:21&lt;00:00,  1.02it/s, valid_loss=4.72]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e2e20571c95e44cb8319f2454d64c2f6"
          }
        },
        "bde4bb315d6041cea08fca0e255a2f11": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b656d24293ee4964900d1126056780e8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "88a3f3e62b2e41df87b8105688801b98": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e2e20571c95e44cb8319f2454d64c2f6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7023d21c52f140a7a119edbcd783743b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_77be2ea8783e4bf9af2971561b25588c",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_5328e71f61414a47a2fa30950d89f1ed",
              "IPY_MODEL_0f246499e9604af58c520e5ffe199bf4"
            ]
          }
        },
        "77be2ea8783e4bf9af2971561b25588c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5328e71f61414a47a2fa30950d89f1ed": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_735905ae59654c9c98d94b7e0504ed4b",
            "_dom_classes": [],
            "description": "train epoch 21: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0f5143ecd56248d689c74a1aaf331687"
          }
        },
        "0f246499e9604af58c520e5ffe199bf4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_6e503e5c5e3f4b0daa0a7cf8736d439b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [03:22&lt;00:00,  4.23it/s, loss=3.58]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6d0abe069d764f249d92bbc113642775"
          }
        },
        "735905ae59654c9c98d94b7e0504ed4b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0f5143ecd56248d689c74a1aaf331687": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6e503e5c5e3f4b0daa0a7cf8736d439b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6d0abe069d764f249d92bbc113642775": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c9f024091d8841d99ad01854fc1cb691": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_cee15e5cc0c94d4188637c25b82e34b3",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_48c22170fbad4a83a7f591e63ff15c68",
              "IPY_MODEL_8d90770b6d794b2fb3c0bafc798c010d"
            ]
          }
        },
        "cee15e5cc0c94d4188637c25b82e34b3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "48c22170fbad4a83a7f591e63ff15c68": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_49584d6498814cf1b8320f973b598ed5",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7333086d247641eba70eced3d21fbc43"
          }
        },
        "8d90770b6d794b2fb3c0bafc798c010d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_fb286ed5e96e4a2981031f0bc03e7ff0",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:21&lt;00:00,  1.03s/it, valid_loss=4.68]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ea2c83790ba14303a461fafeb6a66249"
          }
        },
        "49584d6498814cf1b8320f973b598ed5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7333086d247641eba70eced3d21fbc43": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fb286ed5e96e4a2981031f0bc03e7ff0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ea2c83790ba14303a461fafeb6a66249": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6c8a3569bfa2444681b7a61cf648df9a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_5945abbd4fdf44f2a7f92131710d028d",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_c9a01fcc232c46a78d5407e5731a7436",
              "IPY_MODEL_4b8dcee59f974455876a63334071c639"
            ]
          }
        },
        "5945abbd4fdf44f2a7f92131710d028d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c9a01fcc232c46a78d5407e5731a7436": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_025d0ee242d54468af1440337dcc511c",
            "_dom_classes": [],
            "description": "train epoch 22: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f8c4d86918704308b4b37f6db98b9dc0"
          }
        },
        "4b8dcee59f974455876a63334071c639": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_2bed57b412344be5b4d6bdf5253d1ccb",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [03:19&lt;00:00,  4.36it/s, loss=3.88]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1c758e4db7bf4bed92458d8d580fe470"
          }
        },
        "025d0ee242d54468af1440337dcc511c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f8c4d86918704308b4b37f6db98b9dc0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2bed57b412344be5b4d6bdf5253d1ccb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1c758e4db7bf4bed92458d8d580fe470": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c47bad70e69b46e6a76550d86099af03": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_04f40847ce10479ab8429eb6d15c251d",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_58255918c2b9489aa7738d54992442e0",
              "IPY_MODEL_fba12efe93d64dd1834b871237005eb3"
            ]
          }
        },
        "04f40847ce10479ab8429eb6d15c251d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "58255918c2b9489aa7738d54992442e0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_18c46b9edaf84f7db2669f47679138b9",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5037bebbabd14c61b1169683c7aafe75"
          }
        },
        "fba12efe93d64dd1834b871237005eb3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_39a99785c8364475886de558886ca1fe",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:22&lt;00:00,  1.16s/it, valid_loss=4.7]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ad3f834155174d028b21bc0f6175c232"
          }
        },
        "18c46b9edaf84f7db2669f47679138b9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5037bebbabd14c61b1169683c7aafe75": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "39a99785c8364475886de558886ca1fe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ad3f834155174d028b21bc0f6175c232": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "44f6a968267b446d93cc9abd73e1d927": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_18977684aaba4d51bb098816e2f9556c",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_8285cfee4d4442c4aeeec6913fdb45b3",
              "IPY_MODEL_fd72d7e216cd4dcbb0b2d7c8131737aa"
            ]
          }
        },
        "18977684aaba4d51bb098816e2f9556c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8285cfee4d4442c4aeeec6913fdb45b3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_7925edffcf0a4162aebfa12abdb5f935",
            "_dom_classes": [],
            "description": "train epoch 23: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_18bf824e805d4c7087e3b002994dae54"
          }
        },
        "fd72d7e216cd4dcbb0b2d7c8131737aa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_4b10a63056a444f5805b5d4d52361e48",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [03:20&lt;00:00,  4.68it/s, loss=3.22]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_129e1468e741449ab8edb3b85676884d"
          }
        },
        "7925edffcf0a4162aebfa12abdb5f935": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "18bf824e805d4c7087e3b002994dae54": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4b10a63056a444f5805b5d4d52361e48": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "129e1468e741449ab8edb3b85676884d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9c2bcd73d15948f2a01be41f75da486b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_e0d7c278bc384bbbabf8bff598bb45fa",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_dbd43ba14a7f4597a9fd848396be5bdd",
              "IPY_MODEL_f2c36b539ceb40f28dd6785fab672667"
            ]
          }
        },
        "e0d7c278bc384bbbabf8bff598bb45fa": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "dbd43ba14a7f4597a9fd848396be5bdd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_ce95a0f5ebc343e89ed11139de9c42af",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_51d07c1531674bceb9e7a11edf4b610b"
          }
        },
        "f2c36b539ceb40f28dd6785fab672667": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_a84d24ed971b4040ace8316ee2c95121",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:21&lt;00:00,  1.10s/it, valid_loss=4.67]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3a2a2419b69345ccbc5b89f2630291fb"
          }
        },
        "ce95a0f5ebc343e89ed11139de9c42af": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "51d07c1531674bceb9e7a11edf4b610b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a84d24ed971b4040ace8316ee2c95121": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3a2a2419b69345ccbc5b89f2630291fb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fcc5d624f7e3492899da67b5728d5398": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_00ff3cfd665a4b82a003d24441cb57d9",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_f0b7118b96124c9ab39429390d37c9eb",
              "IPY_MODEL_0fe5e5b67ea64e2ca29c4972d85f800b"
            ]
          }
        },
        "00ff3cfd665a4b82a003d24441cb57d9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f0b7118b96124c9ab39429390d37c9eb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_89b1788bcaae48389f0ec073e54ca6a6",
            "_dom_classes": [],
            "description": "train epoch 24: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4209ea80cd5147e5b585ca90211277a0"
          }
        },
        "0fe5e5b67ea64e2ca29c4972d85f800b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_9e84965f4fb848958abd801776086b59",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [03:19&lt;00:00,  4.14it/s, loss=3.21]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5d74c464f4844f00ab2bf36ede064fd9"
          }
        },
        "89b1788bcaae48389f0ec073e54ca6a6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4209ea80cd5147e5b585ca90211277a0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9e84965f4fb848958abd801776086b59": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5d74c464f4844f00ab2bf36ede064fd9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c8645e6887fa43ef83ad37dd930e4c8a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_62e12316eb3d432184c8edfc9ad2e1a3",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_0b4c2dfacceb4a08be373086c10db2ff",
              "IPY_MODEL_4587798f21d64b8f87e0863c1d05d727"
            ]
          }
        },
        "62e12316eb3d432184c8edfc9ad2e1a3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0b4c2dfacceb4a08be373086c10db2ff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_258f21d530d84ff78567150fc3a9d617",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0506a06b87de44e99bb886f6d4ea41f4"
          }
        },
        "4587798f21d64b8f87e0863c1d05d727": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_6110ad0c90db4ecd83f5a8705b12fe55",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:21&lt;00:00,  1.23s/it, valid_loss=4.6]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3040776c80ef45c4bd35faa073f423c8"
          }
        },
        "258f21d530d84ff78567150fc3a9d617": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0506a06b87de44e99bb886f6d4ea41f4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6110ad0c90db4ecd83f5a8705b12fe55": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3040776c80ef45c4bd35faa073f423c8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "71872ce80eba4f51b0c79916b3541cc4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_1ac63be73eee4d699a10f14e9605be46",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_5b036be977bd4e76ac340219bb40b6e8",
              "IPY_MODEL_8c100a2482f940af91c0123e92c76540"
            ]
          }
        },
        "1ac63be73eee4d699a10f14e9605be46": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5b036be977bd4e76ac340219bb40b6e8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_2f747fa318e94c4090d856aa813aa385",
            "_dom_classes": [],
            "description": "train epoch 25: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_fdfaff780a2644cdb537c0895fcafd7c"
          }
        },
        "8c100a2482f940af91c0123e92c76540": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_72e0c1891ce64c7b84e3a43e31985b85",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [03:20&lt;00:00,  4.17it/s, loss=3.98]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a5b5467ccb3541f39edcb0773a708044"
          }
        },
        "2f747fa318e94c4090d856aa813aa385": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "fdfaff780a2644cdb537c0895fcafd7c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "72e0c1891ce64c7b84e3a43e31985b85": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a5b5467ccb3541f39edcb0773a708044": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c7bfaca199274391866ec46a23bdedf8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_8b028c27e7ad446b982dc3c9b0f0cfaf",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_64f8c8c4d380489cbc5d1fb3adc56674",
              "IPY_MODEL_97d284308f6147aca16faa5495c0c745"
            ]
          }
        },
        "8b028c27e7ad446b982dc3c9b0f0cfaf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "64f8c8c4d380489cbc5d1fb3adc56674": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_d6270c3ab9214303be0ca78062a2716e",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_918e2ba46396421d8267107c37ca1bd1"
          }
        },
        "97d284308f6147aca16faa5495c0c745": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_ae96332e3b564949930c49d38923aeea",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:22&lt;00:00,  1.06s/it, valid_loss=4.63]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5b6a11ddb5464af38a76496a78e41b50"
          }
        },
        "d6270c3ab9214303be0ca78062a2716e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "918e2ba46396421d8267107c37ca1bd1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ae96332e3b564949930c49d38923aeea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5b6a11ddb5464af38a76496a78e41b50": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9e0fa3454b27483da48b8452bda494b1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_5fbf7da508e348a286efdb23061c73df",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_dab65c11274a4141a994cbdb1d353faf",
              "IPY_MODEL_659a14c602ee471e8539004c7be4a1ee"
            ]
          }
        },
        "5fbf7da508e348a286efdb23061c73df": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "dab65c11274a4141a994cbdb1d353faf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_f02af61894434e8a89962bae16df267c",
            "_dom_classes": [],
            "description": "train epoch 26: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_bfab2726ad484a0fab703e9194230ce8"
          }
        },
        "659a14c602ee471e8539004c7be4a1ee": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_9a6fe8997eb44278b644d7ef46bfcc5f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [03:20&lt;00:00,  4.37it/s, loss=3.54]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9e46e7b2d8ed46b5b4f0b5863144e4d9"
          }
        },
        "f02af61894434e8a89962bae16df267c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "bfab2726ad484a0fab703e9194230ce8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9a6fe8997eb44278b644d7ef46bfcc5f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9e46e7b2d8ed46b5b4f0b5863144e4d9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "85fc1be89f0f410fb3c42ece5b3cf753": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_6727b1ce5e77417a84b81300e521d5cf",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_8e497fb09a2b46f38fbdb183f161c11b",
              "IPY_MODEL_0a7b50d35a34478782587d994b92686f"
            ]
          }
        },
        "6727b1ce5e77417a84b81300e521d5cf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8e497fb09a2b46f38fbdb183f161c11b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_dc17c12200dc43d49bdd9ac553ab6aeb",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d5b95639da0a4fdba55db17b88b435c8"
          }
        },
        "0a7b50d35a34478782587d994b92686f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_e384b0da8603465da025b0c23f314c19",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:21&lt;00:00,  1.02it/s, valid_loss=4.6]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e3dca40f33e149f8aa197b24ded041e7"
          }
        },
        "dc17c12200dc43d49bdd9ac553ab6aeb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d5b95639da0a4fdba55db17b88b435c8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e384b0da8603465da025b0c23f314c19": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e3dca40f33e149f8aa197b24ded041e7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e692d404441847719f36d3d0309e91d2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_a05068727adb45318409d4f804d549fd",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_b2938f9f76754fd1b2febeb80dc16a8c",
              "IPY_MODEL_8e41f220be5f4918a3bc6cc324dbc2fd"
            ]
          }
        },
        "a05068727adb45318409d4f804d549fd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b2938f9f76754fd1b2febeb80dc16a8c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_960f12ba9f9e488ea8a9f8cf2ba1dc9b",
            "_dom_classes": [],
            "description": "train epoch 27: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_49818b51fa9640a9b0ba1a7317582f09"
          }
        },
        "8e41f220be5f4918a3bc6cc324dbc2fd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_213f17650c4243f1b28f07a09cde8595",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [03:20&lt;00:00,  4.38it/s, loss=4.13]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_62f2b37c09864529b0ac309a6a99c7fc"
          }
        },
        "960f12ba9f9e488ea8a9f8cf2ba1dc9b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "49818b51fa9640a9b0ba1a7317582f09": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "213f17650c4243f1b28f07a09cde8595": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "62f2b37c09864529b0ac309a6a99c7fc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2598da5e0ff54bac8637fa6b322b6f49": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_5045b3a2cc8746fc959426f6c7c8d826",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_1c45210124674119806875d1414cfefd",
              "IPY_MODEL_242a8edd91f5457f9dc765c9ad9e870e"
            ]
          }
        },
        "5045b3a2cc8746fc959426f6c7c8d826": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1c45210124674119806875d1414cfefd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_d646b9f67a99498aa028208003e1ab40",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_353630a87cda4f3daf9129d68e7f6d1c"
          }
        },
        "242a8edd91f5457f9dc765c9ad9e870e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_0105c914e18246e1876706e38171b0dd",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:21&lt;00:00,  1.00s/it, valid_loss=4.67]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_db4bd9bc210b4e768b45c3910162144c"
          }
        },
        "d646b9f67a99498aa028208003e1ab40": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "353630a87cda4f3daf9129d68e7f6d1c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0105c914e18246e1876706e38171b0dd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "db4bd9bc210b4e768b45c3910162144c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2bfb9df94e8b423ab0969785b947271c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_4212a4f27158486aa794eb4b46a12d4d",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_bbee25cf88d043b79e99e005449acf4c",
              "IPY_MODEL_0240cdee23054544a11ace7688b63b43"
            ]
          }
        },
        "4212a4f27158486aa794eb4b46a12d4d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "bbee25cf88d043b79e99e005449acf4c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_9a315f66abe34d7a86386b95126cda7d",
            "_dom_classes": [],
            "description": "train epoch 28: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_27133a953b624b29824d8dd6378ba94f"
          }
        },
        "0240cdee23054544a11ace7688b63b43": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_a156157825e3484484d0f745594d673a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [03:20&lt;00:00,  4.16it/s, loss=3.55]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1fb61ad0ca0b42e095fb05f3509ed312"
          }
        },
        "9a315f66abe34d7a86386b95126cda7d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "27133a953b624b29824d8dd6378ba94f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a156157825e3484484d0f745594d673a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1fb61ad0ca0b42e095fb05f3509ed312": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "09d496d04a154da89f95ea540b8cf115": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_52aac5b11d3544638464516c6f353355",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_117c2e8916674523a5a97268ef5a20b8",
              "IPY_MODEL_89ccc3cb8fc448b79779585ec99be6f4"
            ]
          }
        },
        "52aac5b11d3544638464516c6f353355": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "117c2e8916674523a5a97268ef5a20b8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_be74d46ede584de6962fcfc9d8b23728",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_907bc2149d3f43a3b04c2f1c4bcf9245"
          }
        },
        "89ccc3cb8fc448b79779585ec99be6f4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_a4a1648055b342719c1929963266b656",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:22&lt;00:00,  1.08s/it, valid_loss=4.64]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_58fba715d9eb4678965a659b3a9f4ada"
          }
        },
        "be74d46ede584de6962fcfc9d8b23728": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "907bc2149d3f43a3b04c2f1c4bcf9245": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a4a1648055b342719c1929963266b656": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "58fba715d9eb4678965a659b3a9f4ada": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c0d46ba996234a28ad1b7b2047de331a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_883f80544a794b24a5f28b97ff6854c4",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_2f7ce35ebda74f5da2a6b8785aac7b90",
              "IPY_MODEL_eaa074ef9e7847cda5a90277b23b14c4"
            ]
          }
        },
        "883f80544a794b24a5f28b97ff6854c4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2f7ce35ebda74f5da2a6b8785aac7b90": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_1ec1cbddb19e44b1bf5893790c6ecae5",
            "_dom_classes": [],
            "description": "train epoch 29: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d750d039398e42259ada18cc4cb9fdf1"
          }
        },
        "eaa074ef9e7847cda5a90277b23b14c4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_757547ec793648478cf460fb235d306d",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [03:21&lt;00:00,  4.10it/s, loss=3.34]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_aaa35e20a09144b28cbe08aa889c2899"
          }
        },
        "1ec1cbddb19e44b1bf5893790c6ecae5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d750d039398e42259ada18cc4cb9fdf1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "757547ec793648478cf460fb235d306d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "aaa35e20a09144b28cbe08aa889c2899": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7cee055816be4525a2cc7cdf391c2002": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_0f82e6453f1d43f7b8c23f02b21dc1e4",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_b579413909b64e1ca7a7f2431c41341b",
              "IPY_MODEL_8b2960106c414ac7bffcc2df2dd291d1"
            ]
          }
        },
        "0f82e6453f1d43f7b8c23f02b21dc1e4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b579413909b64e1ca7a7f2431c41341b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_d423a5e64e19410fa88d3cabb4284cfa",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7429473f500c4495be4f72c55eb68eca"
          }
        },
        "8b2960106c414ac7bffcc2df2dd291d1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_1ac218f042a74c7a8810c20ffea22aed",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:22&lt;00:00,  1.14s/it, valid_loss=4.58]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_934793d5ad1d4274ac5cb99079bf7712"
          }
        },
        "d423a5e64e19410fa88d3cabb4284cfa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7429473f500c4495be4f72c55eb68eca": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1ac218f042a74c7a8810c20ffea22aed": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "934793d5ad1d4274ac5cb99079bf7712": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "29d3831c2f104b4c9cb038ac92998d90": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_91bb99f4b1874158b30034371769cec6",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_b2c53b329f694f3cb580fac8255267d3",
              "IPY_MODEL_7f0c018b1d83430ebfefd96037f91d6a"
            ]
          }
        },
        "91bb99f4b1874158b30034371769cec6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b2c53b329f694f3cb580fac8255267d3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_6925904a6e494b48a605797a79959a45",
            "_dom_classes": [],
            "description": "train epoch 30: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b16ac046fafb4dadace6110f92298a5c"
          }
        },
        "7f0c018b1d83430ebfefd96037f91d6a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_a2b695322a5e4c829f84153480cb4719",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [03:21&lt;00:00,  4.43it/s, loss=3.53]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a9cc669710974e0aa2ad2d8f6a62de3a"
          }
        },
        "6925904a6e494b48a605797a79959a45": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b16ac046fafb4dadace6110f92298a5c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a2b695322a5e4c829f84153480cb4719": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a9cc669710974e0aa2ad2d8f6a62de3a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3d9fc12a26e742dfa7b8b2878aadf79a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_145e27cf13ba4573816997ce390cc208",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_5c5bb78d039b4e0f8eee1f0c881d251b",
              "IPY_MODEL_1d12488ad00b441da2d9290d1c52af10"
            ]
          }
        },
        "145e27cf13ba4573816997ce390cc208": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5c5bb78d039b4e0f8eee1f0c881d251b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_7fb1385de6244e4cb322e67d84d4b6be",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_cbc218b7688e40daa44509d914d56f4b"
          }
        },
        "1d12488ad00b441da2d9290d1c52af10": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_e9f097be855c4d7488bacebee86169d4",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:21&lt;00:00,  1.01it/s, valid_loss=4.66]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f706c17b10974d578517be77084e55f3"
          }
        },
        "7fb1385de6244e4cb322e67d84d4b6be": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "cbc218b7688e40daa44509d914d56f4b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e9f097be855c4d7488bacebee86169d4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f706c17b10974d578517be77084e55f3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "cf5d34476217407585503b24a9511c46": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_53d46061dccd4aebaf00ab8729c83a89",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_c7d901c8bd154dcbabd13ec8a2ced797",
              "IPY_MODEL_a10fe7be794a4e899de39ffd09f4aa26"
            ]
          }
        },
        "53d46061dccd4aebaf00ab8729c83a89": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c7d901c8bd154dcbabd13ec8a2ced797": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a5ecc8be77aa463f86d2caf73d7fe70f",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1ec8f7e5735548fabcef10480b9bbb20"
          }
        },
        "a10fe7be794a4e899de39ffd09f4aa26": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_b2afcb3ce66c43eca41c809dadec8896",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:21&lt;00:00,  1.03s/it, valid_loss=4.59]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3e9ffdca36174cf39be4b90397ff3661"
          }
        },
        "a5ecc8be77aa463f86d2caf73d7fe70f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1ec8f7e5735548fabcef10480b9bbb20": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b2afcb3ce66c43eca41c809dadec8896": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3e9ffdca36174cf39be4b90397ff3661": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c02582c4e9c947a58acbe53247c5f6e5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_a44bb9dc8e44408c9e3def209ad23b1b",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_b3634d05f0e941ab9d9189246a2d53b6",
              "IPY_MODEL_7ab252b683d84694980d8459c43aaebb"
            ]
          }
        },
        "a44bb9dc8e44408c9e3def209ad23b1b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b3634d05f0e941ab9d9189246a2d53b6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_7d64f7eed7894ebe8de2d6ac64b3f3e2",
            "_dom_classes": [],
            "description": "prediction: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 17,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 17,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_86b984fb1a3d4f0bbbf2de552e02cb4c"
          }
        },
        "7ab252b683d84694980d8459c43aaebb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_136b1ed7287a44caa41effe0c22f9437",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 17/17 [00:17&lt;00:00,  1.03s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7d99d3cdee8747d59133b6c3cafda3d5"
          }
        },
        "7d64f7eed7894ebe8de2d6ac64b3f3e2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "86b984fb1a3d4f0bbbf2de552e02cb4c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "136b1ed7287a44caa41effe0c22f9437": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7d99d3cdee8747d59133b6c3cafda3d5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yM44ZqEIpP-s"
      },
      "source": [
        "# **Homework 5 - Sequence-to-sequence**\n",
        "\n",
        "若有任何問題，歡迎來信至助教信箱 ntu-ml-2021spring-ta@googlegroups.com"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3Yf-w71OpP-v"
      },
      "source": [
        "### (4/21 Updates)\n",
        "1. Link to reference [training curves](https://wandb.ai/george0828zhang/hw5.seq2seq.new).\n",
        "\n",
        "### (4/14 Updates) \n",
        "1. Link to tutorial [video](https://youtu.be/htG5WpZVQPU).\n",
        "2. Now defaults to load `\"avg_last_5_checkpoint.pt\"` to generate prediction.\n",
        "3. Expected run time on Colab with Tesla T4\n",
        "\n",
        "|Baseline|Details|Total Time|\n",
        "|-|:-:|:-:|\n",
        "|Simple|2m 15s $\\times$30 epochs|1hr 8m|\n",
        "|Medium|4m $\\times$30 epochs|2hr|\n",
        "|Strong|8m $\\times$30 epochs (backward)<br>+1hr (back-translation)<br>+15m $\\times$30 epochs (forward)|12hr 30m|"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iUN5huOQpP-w"
      },
      "source": [
        "# Sequence-to-Sequence 介紹\n",
        "- 大多數常見的 seq2seq model 為 encoder-decoder model，主要由兩個部分組成，分別是 encoder 和 decoder，而這兩個部可以使用 recurrent neural network (RNN)或 transformer 來實作，主要是用來解決輸入和輸出的長度不一樣的情況\n",
        "- **Encoder** 是將一連串的輸入，如文字、影片、聲音訊號等，編碼為單個向量，這單個向量可以想像為是整個輸入的抽象表示，包含了整個輸入的資訊\n",
        "- **Decoder** 是將 encoder 輸出的單個向量逐步解碼，一次輸出一個結果，直到將最後目標輸出被產生出來為止，每次輸出會影響下一次的輸出，一般會在開頭加入 \"< BOS >\" 來表示開始解碼，會在結尾輸出 \"< EOS >\" 來表示輸出結束\n",
        "\n",
        "\n",
        "![seq2seq](https://i.imgur.com/0zeDyuI.png)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "noP4-48ypP-x"
      },
      "source": [
        "# 作業介紹\n",
        "- 英文翻譯中文\n",
        "  - 輸入： 一句英文 （e.g.\t\ttom is a student .） \n",
        "  - 輸出： 中文翻譯 （e.g. \t\t湯姆 是 個 學生 。）\n",
        "\n",
        "- TODO\n",
        "  - 訓練一個 RNN 模型達到 Seq2seq 翻譯\n",
        "  - 訓練一個 Transformer 大幅提升效能\n",
        "  - 實作 Back-translation 大幅提升效能"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GPut4d5VpP-x"
      },
      "source": [
        "# 下載和引入需要的函式庫"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lXCjGcPYpP-y",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d76f9cca-caef-40fe-9714-13cbb37d4892"
      },
      "source": [
        "!pip install 'torch>=1.6.0' editdistance matplotlib sacrebleu sacremoses sentencepiece tqdm wandb\n",
        "!pip install --upgrade jupyter ipywidgets"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: torch>=1.6.0 in /usr/local/lib/python3.7/dist-packages (1.9.0+cu102)\n",
            "Requirement already satisfied: editdistance in /usr/local/lib/python3.7/dist-packages (0.5.3)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.7/dist-packages (3.2.2)\n",
            "Collecting sacrebleu\n",
            "  Downloading sacrebleu-1.5.1-py3-none-any.whl (54 kB)\n",
            "\u001b[K     |████████████████████████████████| 54 kB 2.7 MB/s \n",
            "\u001b[?25hCollecting sacremoses\n",
            "  Downloading sacremoses-0.0.45-py3-none-any.whl (895 kB)\n",
            "\u001b[K     |████████████████████████████████| 895 kB 15.3 MB/s \n",
            "\u001b[?25hCollecting sentencepiece\n",
            "  Downloading sentencepiece-0.1.96-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.2 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.2 MB 43.6 MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (4.41.1)\n",
            "Collecting wandb\n",
            "  Downloading wandb-0.11.2-py2.py3-none-any.whl (1.8 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.8 MB 24.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch>=1.6.0) (3.7.4.3)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib) (2.8.1)\n",
            "Requirement already satisfied: numpy>=1.11 in /usr/local/lib/python3.7/dist-packages (from matplotlib) (1.19.5)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib) (1.3.1)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib) (2.4.7)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib) (0.10.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from cycler>=0.10->matplotlib) (1.15.0)\n",
            "Collecting portalocker==2.0.0\n",
            "  Downloading portalocker-2.0.0-py2.py3-none-any.whl (11 kB)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses) (1.0.1)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses) (7.1.2)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.7/dist-packages (from sacremoses) (2019.12.20)\n",
            "Collecting docker-pycreds>=0.4.0\n",
            "  Downloading docker_pycreds-0.4.0-py2.py3-none-any.whl (9.0 kB)\n",
            "Collecting shortuuid>=0.5.0\n",
            "  Downloading shortuuid-1.0.1-py3-none-any.whl (7.5 kB)\n",
            "Requirement already satisfied: protobuf>=3.12.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (3.17.3)\n",
            "Collecting GitPython>=1.0.0\n",
            "  Downloading GitPython-3.1.18-py3-none-any.whl (170 kB)\n",
            "\u001b[K     |████████████████████████████████| 170 kB 62.8 MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests<3,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (2.23.0)\n",
            "Collecting subprocess32>=3.5.3\n",
            "  Downloading subprocess32-3.5.4.tar.gz (97 kB)\n",
            "\u001b[K     |████████████████████████████████| 97 kB 9.0 MB/s \n",
            "\u001b[?25hCollecting configparser>=3.8.1\n",
            "  Downloading configparser-5.0.2-py3-none-any.whl (19 kB)\n",
            "Requirement already satisfied: promise<3,>=2.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (2.3)\n",
            "Collecting urllib3>=1.26.5\n",
            "  Downloading urllib3-1.26.6-py2.py3-none-any.whl (138 kB)\n",
            "\u001b[K     |████████████████████████████████| 138 kB 67.2 MB/s \n",
            "\u001b[?25hCollecting pathtools\n",
            "  Downloading pathtools-0.1.2.tar.gz (11 kB)\n",
            "Collecting sentry-sdk>=1.0.0\n",
            "  Downloading sentry_sdk-1.3.1-py2.py3-none-any.whl (133 kB)\n",
            "\u001b[K     |████████████████████████████████| 133 kB 52.8 MB/s \n",
            "\u001b[?25hRequirement already satisfied: PyYAML in /usr/local/lib/python3.7/dist-packages (from wandb) (3.13)\n",
            "Requirement already satisfied: psutil>=5.0.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (5.4.8)\n",
            "Collecting gitdb<5,>=4.0.1\n",
            "  Downloading gitdb-4.0.7-py3-none-any.whl (63 kB)\n",
            "\u001b[K     |████████████████████████████████| 63 kB 2.5 MB/s \n",
            "\u001b[?25hCollecting smmap<5,>=3.0.1\n",
            "  Downloading smmap-4.0.0-py2.py3-none-any.whl (24 kB)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.0.0->wandb) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.0.0->wandb) (2021.5.30)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.0.0->wandb) (2.10)\n",
            "Collecting requests<3,>=2.0.0\n",
            "  Downloading requests-2.26.0-py2.py3-none-any.whl (62 kB)\n",
            "\u001b[K     |████████████████████████████████| 62 kB 1.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.0.0->wandb) (2.0.2)\n",
            "Building wheels for collected packages: subprocess32, pathtools\n",
            "  Building wheel for subprocess32 (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for subprocess32: filename=subprocess32-3.5.4-py3-none-any.whl size=6502 sha256=f2150e5d15b7d04ae02a95f1374a4786b9a23492c88c5a7beebddc37c8617203\n",
            "  Stored in directory: /root/.cache/pip/wheels/50/ca/fa/8fca8d246e64f19488d07567547ddec8eb084e8c0d7a59226a\n",
            "  Building wheel for pathtools (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pathtools: filename=pathtools-0.1.2-py3-none-any.whl size=8806 sha256=95c77975a13a52ddeb360982c48085d9bb4b5139bdc45c717eb4af1bac2bc2cd\n",
            "  Stored in directory: /root/.cache/pip/wheels/3e/31/09/fa59cef12cdcfecc627b3d24273699f390e71828921b2cbba2\n",
            "Successfully built subprocess32 pathtools\n",
            "Installing collected packages: smmap, urllib3, gitdb, subprocess32, shortuuid, sentry-sdk, requests, portalocker, pathtools, GitPython, docker-pycreds, configparser, wandb, sentencepiece, sacremoses, sacrebleu\n",
            "  Attempting uninstall: urllib3\n",
            "    Found existing installation: urllib3 1.24.3\n",
            "    Uninstalling urllib3-1.24.3:\n",
            "      Successfully uninstalled urllib3-1.24.3\n",
            "  Attempting uninstall: requests\n",
            "    Found existing installation: requests 2.23.0\n",
            "    Uninstalling requests-2.23.0:\n",
            "      Successfully uninstalled requests-2.23.0\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "google-colab 1.0.0 requires requests~=2.23.0, but you have requests 2.26.0 which is incompatible.\n",
            "datascience 0.10.6 requires folium==0.2.1, but you have folium 0.8.3 which is incompatible.\u001b[0m\n",
            "Successfully installed GitPython-3.1.18 configparser-5.0.2 docker-pycreds-0.4.0 gitdb-4.0.7 pathtools-0.1.2 portalocker-2.0.0 requests-2.26.0 sacrebleu-1.5.1 sacremoses-0.0.45 sentencepiece-0.1.96 sentry-sdk-1.3.1 shortuuid-1.0.1 smmap-4.0.0 subprocess32-3.5.4 urllib3-1.26.6 wandb-0.11.2\n",
            "Requirement already satisfied: jupyter in /usr/local/lib/python3.7/dist-packages (1.0.0)\n",
            "Requirement already satisfied: ipywidgets in /usr/local/lib/python3.7/dist-packages (7.6.3)\n",
            "Requirement already satisfied: notebook in /usr/local/lib/python3.7/dist-packages (from jupyter) (5.3.1)\n",
            "Requirement already satisfied: jupyter-console in /usr/local/lib/python3.7/dist-packages (from jupyter) (5.2.0)\n",
            "Requirement already satisfied: ipykernel in /usr/local/lib/python3.7/dist-packages (from jupyter) (4.10.1)\n",
            "Requirement already satisfied: nbconvert in /usr/local/lib/python3.7/dist-packages (from jupyter) (5.6.1)\n",
            "Requirement already satisfied: qtconsole in /usr/local/lib/python3.7/dist-packages (from jupyter) (5.1.1)\n",
            "Requirement already satisfied: nbformat>=4.2.0 in /usr/local/lib/python3.7/dist-packages (from ipywidgets) (5.1.3)\n",
            "Requirement already satisfied: widgetsnbextension~=3.5.0 in /usr/local/lib/python3.7/dist-packages (from ipywidgets) (3.5.1)\n",
            "Requirement already satisfied: jupyterlab-widgets>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from ipywidgets) (1.0.0)\n",
            "Requirement already satisfied: ipython>=4.0.0 in /usr/local/lib/python3.7/dist-packages (from ipywidgets) (5.5.0)\n",
            "Requirement already satisfied: traitlets>=4.3.1 in /usr/local/lib/python3.7/dist-packages (from ipywidgets) (5.0.5)\n",
            "Requirement already satisfied: tornado>=4.0 in /usr/local/lib/python3.7/dist-packages (from ipykernel->jupyter) (5.1.1)\n",
            "Requirement already satisfied: jupyter-client in /usr/local/lib/python3.7/dist-packages (from ipykernel->jupyter) (5.3.5)\n",
            "Requirement already satisfied: decorator in /usr/local/lib/python3.7/dist-packages (from ipython>=4.0.0->ipywidgets) (4.4.2)\n",
            "Requirement already satisfied: setuptools>=18.5 in /usr/local/lib/python3.7/dist-packages (from ipython>=4.0.0->ipywidgets) (57.2.0)\n",
            "Requirement already satisfied: simplegeneric>0.8 in /usr/local/lib/python3.7/dist-packages (from ipython>=4.0.0->ipywidgets) (0.8.1)\n",
            "Requirement already satisfied: pygments in /usr/local/lib/python3.7/dist-packages (from ipython>=4.0.0->ipywidgets) (2.6.1)\n",
            "Requirement already satisfied: pickleshare in /usr/local/lib/python3.7/dist-packages (from ipython>=4.0.0->ipywidgets) (0.7.5)\n",
            "Requirement already satisfied: pexpect in /usr/local/lib/python3.7/dist-packages (from ipython>=4.0.0->ipywidgets) (4.8.0)\n",
            "Requirement already satisfied: prompt-toolkit<2.0.0,>=1.0.4 in /usr/local/lib/python3.7/dist-packages (from ipython>=4.0.0->ipywidgets) (1.0.18)\n",
            "Requirement already satisfied: jupyter-core in /usr/local/lib/python3.7/dist-packages (from nbformat>=4.2.0->ipywidgets) (4.7.1)\n",
            "Requirement already satisfied: jsonschema!=2.5.0,>=2.4 in /usr/local/lib/python3.7/dist-packages (from nbformat>=4.2.0->ipywidgets) (2.6.0)\n",
            "Requirement already satisfied: ipython-genutils in /usr/local/lib/python3.7/dist-packages (from nbformat>=4.2.0->ipywidgets) (0.2.0)\n",
            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.7/dist-packages (from prompt-toolkit<2.0.0,>=1.0.4->ipython>=4.0.0->ipywidgets) (0.2.5)\n",
            "Requirement already satisfied: six>=1.9.0 in /usr/local/lib/python3.7/dist-packages (from prompt-toolkit<2.0.0,>=1.0.4->ipython>=4.0.0->ipywidgets) (1.15.0)\n",
            "Requirement already satisfied: terminado>=0.8.1 in /usr/local/lib/python3.7/dist-packages (from notebook->jupyter) (0.10.1)\n",
            "Requirement already satisfied: Send2Trash in /usr/local/lib/python3.7/dist-packages (from notebook->jupyter) (1.7.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.7/dist-packages (from notebook->jupyter) (2.11.3)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.7/dist-packages (from jupyter-client->ipykernel->jupyter) (2.8.1)\n",
            "Requirement already satisfied: pyzmq>=13 in /usr/local/lib/python3.7/dist-packages (from jupyter-client->ipykernel->jupyter) (22.1.0)\n",
            "Requirement already satisfied: ptyprocess in /usr/local/lib/python3.7/dist-packages (from terminado>=0.8.1->notebook->jupyter) (0.7.0)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from jinja2->notebook->jupyter) (2.0.1)\n",
            "Requirement already satisfied: pandocfilters>=1.4.1 in /usr/local/lib/python3.7/dist-packages (from nbconvert->jupyter) (1.4.3)\n",
            "Requirement already satisfied: bleach in /usr/local/lib/python3.7/dist-packages (from nbconvert->jupyter) (3.3.0)\n",
            "Requirement already satisfied: defusedxml in /usr/local/lib/python3.7/dist-packages (from nbconvert->jupyter) (0.7.1)\n",
            "Requirement already satisfied: entrypoints>=0.2.2 in /usr/local/lib/python3.7/dist-packages (from nbconvert->jupyter) (0.3)\n",
            "Requirement already satisfied: testpath in /usr/local/lib/python3.7/dist-packages (from nbconvert->jupyter) (0.5.0)\n",
            "Requirement already satisfied: mistune<2,>=0.8.1 in /usr/local/lib/python3.7/dist-packages (from nbconvert->jupyter) (0.8.4)\n",
            "Requirement already satisfied: webencodings in /usr/local/lib/python3.7/dist-packages (from bleach->nbconvert->jupyter) (0.5.1)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from bleach->nbconvert->jupyter) (21.0)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->bleach->nbconvert->jupyter) (2.4.7)\n",
            "Requirement already satisfied: qtpy in /usr/local/lib/python3.7/dist-packages (from qtconsole->jupyter) (1.9.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NDpfV2i4pP-y",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5060a14b-1ab5-4ea7-f1dd-543d3ae38692"
      },
      "source": [
        "!git clone https://github.com/pytorch/fairseq.git\n",
        "!cd fairseq && git checkout 9a1c497\n",
        "!pip install --upgrade ./fairseq/"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'fairseq'...\n",
            "remote: Enumerating objects: 28917, done.\u001b[K\n",
            "remote: Counting objects: 100% (383/383), done.\u001b[K\n",
            "remote: Compressing objects: 100% (244/244), done.\u001b[K\n",
            "remote: Total 28917 (delta 180), reused 296 (delta 132), pack-reused 28534\u001b[K\n",
            "Receiving objects: 100% (28917/28917), 12.24 MiB | 24.76 MiB/s, done.\n",
            "Resolving deltas: 100% (21635/21635), done.\n",
            "Note: checking out '9a1c497'.\n",
            "\n",
            "You are in 'detached HEAD' state. You can look around, make experimental\n",
            "changes and commit them, and you can discard any commits you make in this\n",
            "state without impacting any branches by performing another checkout.\n",
            "\n",
            "If you want to create a new branch to retain commits you create, you may\n",
            "do so (now or later) by using -b with the checkout command again. Example:\n",
            "\n",
            "  git checkout -b <new-branch-name>\n",
            "\n",
            "HEAD is now at 9a1c4970 Make Hydra logging work with DDP (#1568)\n",
            "Processing ./fairseq\n",
            "\u001b[33m  DEPRECATION: A future pip version will change local packages to be built in-place without first copying to a temporary directory. We recommend you use --use-feature=in-tree-build to test your packages with this new behavior before it becomes the default.\n",
            "   pip 21.3 will remove support for this functionality. You can find discussion regarding this at https://github.com/pypa/pip/issues/7555.\u001b[0m\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Installing backend dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "    Preparing wheel metadata ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from fairseq==1.0.0a0+9a1c497) (4.41.1)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.7/dist-packages (from fairseq==1.0.0a0+9a1c497) (2019.12.20)\n",
            "Requirement already satisfied: cffi in /usr/local/lib/python3.7/dist-packages (from fairseq==1.0.0a0+9a1c497) (1.14.6)\n",
            "Collecting omegaconf<2.1\n",
            "  Downloading omegaconf-2.0.6-py3-none-any.whl (36 kB)\n",
            "Requirement already satisfied: sacrebleu>=1.4.12 in /usr/local/lib/python3.7/dist-packages (from fairseq==1.0.0a0+9a1c497) (1.5.1)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from fairseq==1.0.0a0+9a1c497) (1.19.5)\n",
            "Requirement already satisfied: cython in /usr/local/lib/python3.7/dist-packages (from fairseq==1.0.0a0+9a1c497) (0.29.23)\n",
            "Collecting hydra-core<1.1\n",
            "  Downloading hydra_core-1.0.7-py3-none-any.whl (123 kB)\n",
            "\u001b[K     |████████████████████████████████| 123 kB 10.8 MB/s \n",
            "\u001b[?25hRequirement already satisfied: torch in /usr/local/lib/python3.7/dist-packages (from fairseq==1.0.0a0+9a1c497) (1.9.0+cu102)\n",
            "Collecting antlr4-python3-runtime==4.8\n",
            "  Downloading antlr4-python3-runtime-4.8.tar.gz (112 kB)\n",
            "\u001b[K     |████████████████████████████████| 112 kB 55.8 MB/s \n",
            "\u001b[?25hRequirement already satisfied: importlib-resources in /usr/local/lib/python3.7/dist-packages (from hydra-core<1.1->fairseq==1.0.0a0+9a1c497) (5.2.0)\n",
            "Collecting PyYAML>=5.1.*\n",
            "  Downloading PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl (636 kB)\n",
            "\u001b[K     |████████████████████████████████| 636 kB 43.8 MB/s \n",
            "\u001b[?25hRequirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from omegaconf<2.1->fairseq==1.0.0a0+9a1c497) (3.7.4.3)\n",
            "Requirement already satisfied: portalocker==2.0.0 in /usr/local/lib/python3.7/dist-packages (from sacrebleu>=1.4.12->fairseq==1.0.0a0+9a1c497) (2.0.0)\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.7/dist-packages (from cffi->fairseq==1.0.0a0+9a1c497) (2.20)\n",
            "Requirement already satisfied: zipp>=3.1.0 in /usr/local/lib/python3.7/dist-packages (from importlib-resources->hydra-core<1.1->fairseq==1.0.0a0+9a1c497) (3.5.0)\n",
            "Building wheels for collected packages: fairseq, antlr4-python3-runtime\n",
            "  Building wheel for fairseq (PEP 517) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for fairseq: filename=fairseq-1.0.0a0+9a1c497-cp37-cp37m-linux_x86_64.whl size=2851258 sha256=2bcd7f7b6a9df9d6e1a4a498c6cfbf348b2c42c9d9bd562461a2b84d90384aa8\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-21sp2m4m/wheels/7c/35/80/edbd520a1a7e615df007002aeea9f6bf5f3c8f9243e072f6ce\n",
            "  Building wheel for antlr4-python3-runtime (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for antlr4-python3-runtime: filename=antlr4_python3_runtime-4.8-py3-none-any.whl size=141229 sha256=cc9d9b8cc5d5270ba6e4382b0514ad04e7fe68a7490e07e76b7f4c792c192657\n",
            "  Stored in directory: /root/.cache/pip/wheels/ca/33/b7/336836125fc9bb4ceaa4376d8abca10ca8bc84ddc824baea6c\n",
            "Successfully built fairseq antlr4-python3-runtime\n",
            "Installing collected packages: PyYAML, omegaconf, antlr4-python3-runtime, hydra-core, fairseq\n",
            "  Attempting uninstall: PyYAML\n",
            "    Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "Successfully installed PyYAML-5.4.1 antlr4-python3-runtime-4.8 fairseq-1.0.0a0+9a1c497 hydra-core-1.0.7 omegaconf-2.0.6\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_9mPzfqapP-z"
      },
      "source": [
        "import sys\n",
        "import pdb\n",
        "import pprint\n",
        "import logging\n",
        "import os\n",
        "import random\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "from torch.utils import data\n",
        "import numpy as np\n",
        "import tqdm.auto as tqdm\n",
        "from pathlib import Path\n",
        "from argparse import Namespace\n",
        "from fairseq import utils\n",
        "\n",
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "O5z6H8w-pP-z"
      },
      "source": [
        "# 設定種子"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FTR4HiC4pP-0"
      },
      "source": [
        "seed = 73\n",
        "random.seed(seed)\n",
        "torch.manual_seed(seed)\n",
        "if torch.cuda.is_available():\n",
        "    torch.cuda.manual_seed(seed)\n",
        "    torch.cuda.manual_seed_all(seed)  \n",
        "np.random.seed(seed)  \n",
        "torch.backends.cudnn.benchmark = False\n",
        "torch.backends.cudnn.deterministic = True"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_AzgJr-BpP-0"
      },
      "source": [
        "# 資料集介紹\n",
        "\n",
        "## 英轉繁雙語資料\n",
        "* [TED2020](#reimers-2020-multilingual-sentence-bert)\n",
        "    - 原始資料量: 398,066句    \n",
        "    - 處理後資料: 393,980句\n",
        "    \n",
        "\n",
        "## 測試資料\n",
        "- 資料量: 4,000句\n",
        "- **中文部分不公開，提供的檔案為假翻譯，全部都是句點。**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cnBKxzh7pP-1"
      },
      "source": [
        "# 資料下載"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "G0hYaw_0pP-1"
      },
      "source": [
        "### 安裝megatools (optional)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "usvzjowWpP-1"
      },
      "source": [
        "#!apt-get install megatools"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FbmGrn2qpP-2"
      },
      "source": [
        "## 下載檔案並解壓縮"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "s3saCXi00Qwz"
      },
      "source": [
        "下载数据并解压"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "R4lrO30W0P88",
        "outputId": "3a9a54f6-3134-45d9-eba6-af2b19df2fcc"
      },
      "source": [
        "!gdown --id '15f5ZghJJ1AQ9rARr4XDNCQcfxGXtzWyt' --output ted2020.tgz\n",
        "!gdown --id '18WjD1jP5p7g6E3YuAJs6PaAADvhKEbuq' --output test.tgz\n",
        "!tar -xvf ted2020.tgz \n",
        "!tar -xvf test.tgz "
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading...\n",
            "From: https://drive.google.com/uc?id=15f5ZghJJ1AQ9rARr4XDNCQcfxGXtzWyt\n",
            "To: /content/ted2020.tgz\n",
            "28.6MB [00:00, 108MB/s] \n",
            "Downloading...\n",
            "From: https://drive.google.com/uc?id=18WjD1jP5p7g6E3YuAJs6PaAADvhKEbuq\n",
            "To: /content/test.tgz\n",
            "100% 150k/150k [00:00<00:00, 55.6MB/s]\n",
            "raw.en\n",
            "raw.zh\n",
            "test.en\n",
            "test.zh\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nXLHr4Li4QMi"
      },
      "source": [
        "将数据移到给定目录下"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VOgdu-Zl2re5",
        "outputId": "db0a7db6-bce1-4d67-ca96-3e5db5fec342"
      },
      "source": [
        "data_dir = './DATA/rawdata'\n",
        "dataset_name = 'ted2020'\n",
        "prefix = Path(data_dir).absolute() / dataset_name\n",
        "prefix.mkdir(parents=True, exist_ok=True)\n",
        "print(prefix)\n",
        "!mv {'raw.en'} {prefix/'train_dev.raw.en'}\n",
        "!mv {'raw.zh'} {prefix/'train_dev.raw.zh'}\n",
        "!mv {'test.en'} {prefix/'test.raw.en'}\n",
        "!mv {'test.zh'} {prefix/'test.raw.zh'}"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/DATA/rawdata/ted2020\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VRYNJLjXpP-2",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3a91eefe-e5f9-442d-8c6f-a1d88ccb479b"
      },
      "source": [
        "'''\n",
        "data_dir = './DATA/rawdata'\n",
        "dataset_name = 'ted2020'\n",
        "urls = (\n",
        "    '\"https://onedrive.live.com/download?cid=3E549F3B24B238B4&resid=3E549F3B24B238B4%214989&authkey=AGgQ-DaR8eFSl1A\"', \n",
        "    '\"https://onedrive.live.com/download?cid=3E549F3B24B238B4&resid=3E549F3B24B238B4%214987&authkey=AA4qP_azsicwZZM\"',\n",
        "# # If the above links die, use the following instead. \n",
        "#     \"https://www.csie.ntu.edu.tw/~r09922057/ML2021-hw5/ted2020.tgz\",\n",
        "#     \"https://www.csie.ntu.edu.tw/~r09922057/ML2021-hw5/test.tgz\",\n",
        "# # If the above links die, use the following instead. \n",
        "#     \"https://mega.nz/#!vEcTCISJ!3Rw0eHTZWPpdHBTbQEqBDikDEdFPr7fI8WxaXK9yZ9U\",\n",
        "#     \"https://mega.nz/#!zNcnGIoJ!oPJX9AvVVs11jc0SaK6vxP_lFUNTkEcK2WbxJpvjU5Y\",\n",
        "#      \"https://drive.google.com/file/d/15f5ZghJJ1AQ9rARr4XDNCQcfxGXtzWyt/view?usp=sharing\",\n",
        "#      \"https://drive.google.com/file/d/18WjD1jP5p7g6E3YuAJs6PaAADvhKEbuq/view?usp=sharing\"\n",
        ")\n",
        "file_names = (\n",
        "    'ted2020', # train & dev\n",
        "    'test', # test\n",
        ")\n",
        "prefix = Path(data_dir).absolute() / dataset_name\n",
        "\n",
        "prefix.mkdir(parents=True, exist_ok=True)\n",
        "print(prefix)\n",
        "# for u, f in zip(urls, file_names):\n",
        "#     path = prefix/f\n",
        "#     if not path.exists():\n",
        "#         if 'mega' in u:\n",
        "#             !megadl {u} --path {path}\n",
        "#         else:\n",
        "#             !wget {u} -O {path}\n",
        "#     print(\"********************\")\n",
        "#     print(path.suffix,path,prefix)\n",
        "#     if path.suffix == \".tgz\":\n",
        "#         !tar -xvf {path} -C {prefix}\n",
        "#     elif path.suffix == \".zip\":\n",
        "#         !unzip -o {path} -d {prefix}\n",
        "!mv {prefix/'raw.en'} {prefix/'train_dev.raw.en'}\n",
        "!mv {prefix/'raw.zh'} {prefix/'train_dev.raw.zh'}\n",
        "!mv {prefix/'test.en'} {prefix/'test.raw.en'}\n",
        "!mv {prefix/'test.zh'} {prefix/'test.raw.zh'}\n",
        "'''"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/DATA/rawdata/ted2020\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NYgMcc8OpP-4"
      },
      "source": [
        "## 設定語言"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YH1ETbe_pP-4"
      },
      "source": [
        "src_lang = 'en'\n",
        "tgt_lang = 'zh'\n",
        "\n",
        "data_prefix = f'{prefix}/train_dev.raw'\n",
        "test_prefix = f'{prefix}/test.raw'"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7CKQcy_KpP-4",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "201f26db-7d32-436d-d0d9-2fc5ac67eb78"
      },
      "source": [
        "!head {data_prefix+'.'+src_lang} -n 5\n",
        "!head {data_prefix+'.'+tgt_lang} -n 5"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Thank you so much, Chris.\n",
            "And it's truly a great honor to have the opportunity to come to this stage twice; I'm extremely grateful.\n",
            "I have been blown away by this conference, and I want to thank all of you for the many nice comments about what I had to say the other night.\n",
            "And I say that sincerely, partly because  I need that.\n",
            "Put yourselves in my position.\n",
            "非常謝謝你，克里斯。能有這個機會第二度踏上這個演講台\n",
            "真是一大榮幸。我非常感激。\n",
            "這個研討會給我留下了極為深刻的印象，我想感謝大家 對我之前演講的好評。\n",
            "我是由衷的想這麼說，有部份原因是因為 —— 我真的有需要!\n",
            "請你們設身處地為我想一想！\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "g6hDMXHBpP-5"
      },
      "source": [
        "## 檔案前處理"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iGSKDmDWpP-5"
      },
      "source": [
        "import re\n",
        "\n",
        "def strQ2B(ustring):\n",
        "    \"\"\"把字串全形轉半形\"\"\"\n",
        "    # 參考來源:https://ithelp.ithome.com.tw/articles/10233122\n",
        "    ss = []\n",
        "    for s in ustring:\n",
        "        rstring = \"\"\n",
        "        for uchar in s:\n",
        "            inside_code = ord(uchar)\n",
        "            if inside_code == 12288:  # 全形空格直接轉換\n",
        "                inside_code = 32\n",
        "            elif (inside_code >= 65281 and inside_code <= 65374):  # 全形字元（除空格）根據關係轉化\n",
        "                inside_code -= 65248\n",
        "            rstring += chr(inside_code)\n",
        "        ss.append(rstring)\n",
        "    return ''.join(ss)\n",
        "                \n",
        "def clean_s(s, lang):\n",
        "    if lang == 'en':\n",
        "        s = re.sub(r\"\\([^()]*\\)\", \"\", s) # remove ([text])\n",
        "        s = s.replace('-', '') # remove '-'\n",
        "        s = re.sub('([.,;!?()\\\"])', r' \\1 ', s) # keep punctuation\n",
        "    elif lang == 'zh':\n",
        "        s = strQ2B(s) # Q2B\n",
        "        s = re.sub(r\"\\([^()]*\\)\", \"\", s) # remove ([text])\n",
        "        s = s.replace(' ', '')\n",
        "        s = s.replace('—', '')\n",
        "        s = s.replace('“', '\"')\n",
        "        s = s.replace('”', '\"')\n",
        "        s = s.replace('_', '')\n",
        "        s = re.sub('([。,;!?()\\\"~「」])', r' \\1 ', s) # keep punctuation\n",
        "    s = ' '.join(s.strip().split())\n",
        "    return s\n",
        "\n",
        "def len_s(s, lang):\n",
        "    if lang == 'zh':\n",
        "        return len(s)\n",
        "    return len(s.split())\n",
        "\n",
        "def clean_corpus(prefix, l1, l2, ratio=9, max_len=1000, min_len=1):\n",
        "    if Path(f'{prefix}.clean.{l1}').exists() and Path(f'{prefix}.clean.{l2}').exists():\n",
        "        print(f'{prefix}.clean.{l1} & {l2} exists. skipping clean.')\n",
        "        return\n",
        "    with open(f'{prefix}.{l1}', 'r') as l1_in_f:\n",
        "        with open(f'{prefix}.{l2}', 'r') as l2_in_f:\n",
        "            with open(f'{prefix}.clean.{l1}', 'w') as l1_out_f:\n",
        "                with open(f'{prefix}.clean.{l2}', 'w') as l2_out_f:\n",
        "                    for s1 in l1_in_f:\n",
        "                        s1 = s1.strip()\n",
        "                        s2 = l2_in_f.readline().strip()\n",
        "                        s1 = clean_s(s1, l1)\n",
        "                        s2 = clean_s(s2, l2)\n",
        "                        s1_len = len_s(s1, l1)\n",
        "                        s2_len = len_s(s2, l2)\n",
        "                        if min_len > 0: # remove short sentence\n",
        "                            if s1_len < min_len or s2_len < min_len:\n",
        "                                continue\n",
        "                        if max_len > 0: # remove long sentence\n",
        "                            if s1_len > max_len or s2_len > max_len:\n",
        "                                continue\n",
        "                        if ratio > 0: # remove by ratio of length\n",
        "                            if s1_len/s2_len > ratio or s2_len/s1_len > ratio:\n",
        "                                continue\n",
        "                        print(s1, file=l1_out_f)\n",
        "                        print(s2, file=l2_out_f)"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S-5Q8DfepP-7"
      },
      "source": [
        "clean_corpus(data_prefix, src_lang, tgt_lang)\n",
        "clean_corpus(test_prefix, src_lang, tgt_lang, ratio=-1, min_len=-1, max_len=-1)"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dkxg7njTpP-8",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "56d40698-eef4-451b-9fca-57f0110492da"
      },
      "source": [
        "!head {data_prefix+'.clean.'+src_lang} -n 5\n",
        "!head {data_prefix+'.clean.'+tgt_lang} -n 5"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Thank you so much , Chris .\n",
            "And it's truly a great honor to have the opportunity to come to this stage twice ; I'm extremely grateful .\n",
            "I have been blown away by this conference , and I want to thank all of you for the many nice comments about what I had to say the other night .\n",
            "And I say that sincerely , partly because I need that .\n",
            "Put yourselves in my position .\n",
            "非常謝謝你 , 克里斯 。 能有這個機會第二度踏上這個演講台\n",
            "真是一大榮幸 。 我非常感激 。\n",
            "這個研討會給我留下了極為深刻的印象 , 我想感謝大家對我之前演講的好評 。\n",
            "我是由衷的想這麼說 , 有部份原因是因為我真的有需要 !\n",
            "請你們設身處地為我想一想 !\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6DP6D9ZtpP-9"
      },
      "source": [
        "## 切出 train/valid set"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "txbQd_E7pP-9"
      },
      "source": [
        "valid_ratio = 0.01 # 3000~4000句就夠了\n",
        "train_ratio = 1 - valid_ratio"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q6GdEhJOpP-9"
      },
      "source": [
        "if (prefix/f'train.clean.{src_lang}').exists() \\\n",
        "and (prefix/f'train.clean.{tgt_lang}').exists() \\\n",
        "and (prefix/f'valid.clean.{src_lang}').exists() \\\n",
        "and (prefix/f'valid.clean.{tgt_lang}').exists():\n",
        "    print(f'train/valid splits exists. skipping split.')\n",
        "else:\n",
        "    line_num = sum(1 for line in open(f'{data_prefix}.clean.{src_lang}'))\n",
        "    labels = list(range(line_num))\n",
        "    random.shuffle(labels)\n",
        "    for lang in [src_lang, tgt_lang]:\n",
        "        train_f = open(os.path.join(data_dir, dataset_name, f'train.clean.{lang}'), 'w')\n",
        "        valid_f = open(os.path.join(data_dir, dataset_name, f'valid.clean.{lang}'), 'w')\n",
        "        count = 0\n",
        "        for line in open(f'{data_prefix}.clean.{lang}', 'r'):\n",
        "            if labels[count]/line_num < train_ratio:\n",
        "                train_f.write(line)\n",
        "            else:\n",
        "                valid_f.write(line)\n",
        "            count += 1\n",
        "        train_f.close()\n",
        "        valid_f.close()"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A0NObpwUpP--"
      },
      "source": [
        "## Subword Units \n",
        "翻譯存在的一大問題是未登錄詞(out of vocabulary)，可以使用 subword units 作為斷詞單位來解決。\n",
        "- 使用 [sentencepiece](#kudo-richardson-2018-sentencepiece) 套件\n",
        "- 用 unigram 或 byte-pair encoding (BPE)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uxSUnKzdpP--"
      },
      "source": [
        "import sentencepiece as spm\n",
        "vocab_size = 8000\n",
        "if (prefix/f'spm{vocab_size}.model').exists():\n",
        "    print(f'{prefix}/spm{vocab_size}.model exists. skipping spm_train.')\n",
        "else:\n",
        "    spm.SentencePieceTrainer.train(\n",
        "        input=','.join([f'{prefix}/train.clean.{src_lang}',\n",
        "                        f'{prefix}/valid.clean.{src_lang}',\n",
        "                        f'{prefix}/train.clean.{tgt_lang}',\n",
        "                        f'{prefix}/valid.clean.{tgt_lang}']),\n",
        "        model_prefix=prefix/f'spm{vocab_size}',\n",
        "        vocab_size=vocab_size,\n",
        "        character_coverage=1,\n",
        "        model_type='unigram', # 'bpe' 也可\n",
        "        input_sentence_size=1e6,\n",
        "        shuffle_input_sentence=True,\n",
        "        normalization_rule_name='nmt_nfkc_cf',\n",
        "    )"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9BG7sTUBpP--"
      },
      "source": [
        "spm_model = spm.SentencePieceProcessor(model_file=str(prefix/f'spm{vocab_size}.model'))\n",
        "in_tag = {\n",
        "    'train': 'train.clean',\n",
        "    'valid': 'valid.clean',\n",
        "    'test': 'test.raw.clean',\n",
        "}\n",
        "for split in ['train', 'valid', 'test']:\n",
        "    for lang in [src_lang, tgt_lang]:\n",
        "        out_path = prefix/f'{split}.{lang}'\n",
        "        if out_path.exists():\n",
        "            print(f\"{out_path} exists. skipping spm_encode.\")\n",
        "        else:\n",
        "            with open(prefix/f'{split}.{lang}', 'w') as out_f:\n",
        "                with open(prefix/f'{in_tag[split]}.{lang}', 'r') as in_f:\n",
        "                    for line in in_f:\n",
        "                        line = line.strip()\n",
        "                        tok = spm_model.encode(line, out_type=str)\n",
        "                        print(' '.join(tok), file=out_f)"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HqI-xuo0pP-_",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b2d11160-0f14-49b1-eca7-2e46f9a336aa"
      },
      "source": [
        "!head {data_dir+'/'+dataset_name+'/train.'+src_lang} -n 5\n",
        "!head {data_dir+'/'+dataset_name+'/train.'+tgt_lang} -n 5"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "▁thank ▁you ▁so ▁much ▁, ▁chris ▁.\n",
            "▁and ▁it ' s ▁ t ru ly ▁a ▁great ▁ho n or ▁to ▁have ▁the ▁ op port un ity ▁to ▁come ▁to ▁this ▁st age ▁ t wi ce ▁; ▁i ' m ▁ex t re me ly ▁gr ate ful ▁.\n",
            "▁i ▁have ▁been ▁ bl ow n ▁away ▁by ▁this ▁con fer ence ▁, ▁and ▁i ▁want ▁to ▁thank ▁all ▁of ▁you ▁for ▁the ▁many ▁ ni ce ▁ com ment s ▁about ▁what ▁i ▁had ▁to ▁say ▁the ▁other ▁night ▁.\n",
            "▁and ▁i ▁say ▁that ▁since re ly ▁, ▁part ly ▁because ▁i ▁need ▁that ▁.\n",
            "▁put ▁your s el ve s ▁in ▁my ▁po s ition ▁.\n",
            "▁ 非常 謝 謝 你 ▁, ▁ 克 里 斯 ▁。 ▁ 能 有 這個 機會 第二 度 踏 上 這個 演講 台\n",
            "▁ 真 是 一 大 榮 幸 ▁。 ▁我 非常 感 激 ▁。\n",
            "▁這個 研 討 會 給我 留 下 了 極 為 深 刻 的 印 象 ▁, ▁我想 感 謝 大家 對我 之前 演講 的 好 評 ▁。\n",
            "▁我 是由 衷 的 想 這麼 說 ▁, ▁有 部份 原因 是因為 我 真的 有 需要 ▁!\n",
            "▁ 請 你們 設 身 處 地 為 我想 一 想 ▁!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OfVHU-YppP-_"
      },
      "source": [
        "## 用 fairseq 將資料轉為 binary"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M0f0diELpP-_",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e5991855-69ee-44b9-bfbb-1b56c41f47d0"
      },
      "source": [
        "binpath = Path('./DATA/data-bin', dataset_name)\n",
        "if binpath.exists():\n",
        "    print(binpath, \"exists, will not overwrite!\")\n",
        "else:\n",
        "    !python -m fairseq_cli.preprocess \\\n",
        "        --source-lang {src_lang}\\\n",
        "        --target-lang {tgt_lang}\\\n",
        "        --trainpref {prefix/'train'}\\\n",
        "        --validpref {prefix/'valid'}\\\n",
        "        --testpref {prefix/'test'}\\\n",
        "        --destdir {binpath}\\\n",
        "        --joined-dictionary\\\n",
        "        --workers 2"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-08-03 02:26:16 | INFO | fairseq_cli.preprocess | Namespace(align_suffix=None, alignfile=None, all_gather_list_size=16384, azureml_logging=False, bf16=False, bpe=None, cpu=False, criterion='cross_entropy', dataset_impl='mmap', destdir='DATA/data-bin/ted2020', empty_cache_freq=0, fp16=False, fp16_init_scale=128, fp16_no_flatten_grads=False, fp16_scale_tolerance=0.0, fp16_scale_window=None, joined_dictionary=True, log_format=None, log_interval=100, lr_scheduler='fixed', memory_efficient_bf16=False, memory_efficient_fp16=False, min_loss_scale=0.0001, model_parallel_size=1, no_progress_bar=False, nwordssrc=-1, nwordstgt=-1, only_source=False, optimizer=None, padding_factor=8, profile=False, quantization_config_path=None, reset_logging=False, scoring='bleu', seed=1, source_lang='en', srcdict=None, suppress_crashes=False, target_lang='zh', task='translation', tensorboard_logdir=None, testpref='/content/DATA/rawdata/ted2020/test', tgtdict=None, threshold_loss_scale=None, thresholdsrc=0, thresholdtgt=0, tokenizer=None, tpu=False, trainpref='/content/DATA/rawdata/ted2020/train', user_dir=None, validpref='/content/DATA/rawdata/ted2020/valid', wandb_project=None, workers=2)\n",
            "2021-08-03 02:27:26 | INFO | fairseq_cli.preprocess | [en] Dictionary: 8000 types\n",
            "2021-08-03 02:28:23 | INFO | fairseq_cli.preprocess | [en] /content/DATA/rawdata/ted2020/train.en: 390041 sents, 12215608 tokens, 0.0% replaced by <unk>\n",
            "2021-08-03 02:28:23 | INFO | fairseq_cli.preprocess | [en] Dictionary: 8000 types\n",
            "2021-08-03 02:28:23 | INFO | fairseq_cli.preprocess | [en] /content/DATA/rawdata/ted2020/valid.en: 3939 sents, 122560 tokens, 0.0% replaced by <unk>\n",
            "2021-08-03 02:28:23 | INFO | fairseq_cli.preprocess | [en] Dictionary: 8000 types\n",
            "2021-08-03 02:28:24 | INFO | fairseq_cli.preprocess | [en] /content/DATA/rawdata/ted2020/test.en: 4000 sents, 122894 tokens, 0.0% replaced by <unk>\n",
            "2021-08-03 02:28:24 | INFO | fairseq_cli.preprocess | [zh] Dictionary: 8000 types\n",
            "2021-08-03 02:29:14 | INFO | fairseq_cli.preprocess | [zh] /content/DATA/rawdata/ted2020/train.zh: 390041 sents, 9533185 tokens, 0.0% replaced by <unk>\n",
            "2021-08-03 02:29:14 | INFO | fairseq_cli.preprocess | [zh] Dictionary: 8000 types\n",
            "2021-08-03 02:29:15 | INFO | fairseq_cli.preprocess | [zh] /content/DATA/rawdata/ted2020/valid.zh: 3939 sents, 96124 tokens, 0.0052% replaced by <unk>\n",
            "2021-08-03 02:29:15 | INFO | fairseq_cli.preprocess | [zh] Dictionary: 8000 types\n",
            "2021-08-03 02:29:15 | INFO | fairseq_cli.preprocess | [zh] /content/DATA/rawdata/ted2020/test.zh: 4000 sents, 8000 tokens, 0.0% replaced by <unk>\n",
            "2021-08-03 02:29:15 | INFO | fairseq_cli.preprocess | Wrote preprocessed data to DATA/data-bin/ted2020\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_hJ6z2ARpP_A"
      },
      "source": [
        "# 實驗的參數設定表"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_ivt013cpP_A"
      },
      "source": [
        "config = Namespace(\n",
        "    datadir = \"./DATA/data-bin/ted2020\",\n",
        "    savedir = \"./checkpoints/rnn\",\n",
        "    source_lang = \"en\",\n",
        "    target_lang = \"zh\",\n",
        "    \n",
        "    # cpu threads when fetching & processing data.\n",
        "    num_workers=2,  \n",
        "    # batch size in terms of tokens. gradient accumulation increases the effective batchsize.\n",
        "    max_tokens=8192,\n",
        "    accum_steps=2,\n",
        "    \n",
        "    # the lr s calculated from Noam lr scheduler. you can tune the maximum lr by this factor.\n",
        "    lr_factor=2.,\n",
        "    lr_warmup=4000,\n",
        "    \n",
        "    # clipping gradient norm helps alleviate gradient exploding\n",
        "    clip_norm=1.0,\n",
        "    \n",
        "    # maximum epochs for training\n",
        "    max_epoch=30,\n",
        "    start_epoch=1,\n",
        "    \n",
        "    # beam size for beam search\n",
        "    beam=5, \n",
        "    # generate sequences of maximum length ax + b, where x is the source length\n",
        "    max_len_a=1.2, \n",
        "    max_len_b=10,\n",
        "    # when decoding, post process sentence by removing sentencepiece symbols.\n",
        "    post_process = \"sentencepiece\",\n",
        "    \n",
        "    # checkpoints\n",
        "    keep_last_epochs=5,\n",
        "    resume=None, # if resume from checkpoint name (under config.savedir)\n",
        "    \n",
        "    # logging\n",
        "    use_wandb=False,\n",
        ")"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OdElT9hhpP_A"
      },
      "source": [
        "# Logging\n",
        "- logging 套件紀錄一般訊息\n",
        "- wandb 紀錄續練過程 loss, bleu, model weight 等等"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": true,
        "id": "CgMpRY4XpP_B"
      },
      "source": [
        "logging.basicConfig(\n",
        "    format=\"%(asctime)s | %(levelname)s | %(name)s | %(message)s\",\n",
        "    datefmt=\"%Y-%m-%d %H:%M:%S\",\n",
        "    level=\"INFO\", # \"DEBUG\" \"WARNING\" \"ERROR\"\n",
        "    stream=sys.stdout,\n",
        ")\n",
        "proj = \"hw5.seq2seq\"\n",
        "logger = logging.getLogger(proj)\n",
        "if config.use_wandb:\n",
        "    import wandb\n",
        "    wandb.init(project=proj, name=Path(config.savedir).stem, config=config)"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CIkLO7OqpP_B"
      },
      "source": [
        "# CUDA環境"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "crJIFtYYpP_B",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "edd3de4f-469a-42c5-b161-2f9ae42c920c"
      },
      "source": [
        "cuda_env = utils.CudaEnvironment()\n",
        "utils.CudaEnvironment.pretty_print_cuda_env_list([cuda_env])\n",
        "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-08-03 02:30:53 | INFO | fairseq.utils | ***********************CUDA enviroments for all 1 workers***********************\n",
            "2021-08-03 02:30:53 | INFO | fairseq.utils | rank   0: capabilities =  7.5  ; total memory = 14.756 GB ; name = Tesla T4                                \n",
            "2021-08-03 02:30:53 | INFO | fairseq.utils | ***********************CUDA enviroments for all 1 workers***********************\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "l0FdigYIpP_B"
      },
      "source": [
        "# 讀取資料集"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fBE-S_3tpP_C"
      },
      "source": [
        "## 借用 fairseq 的 TranslationTask\n",
        "* 用來讀進上面 binarized 的檔案\n",
        "* 有現成的 data iterator (dataloader)\n",
        "* 字典 task.source_dictionary 和 task.target_dictionary 也很好用 \n",
        "* 有實做 beam search"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A-Do4G8GpP_C",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "74ad85e3-128f-4010-ae3f-59c96f5c548e"
      },
      "source": [
        "from fairseq.tasks.translation import TranslationConfig, TranslationTask\n",
        "\n",
        "## setup task\n",
        "task_cfg = TranslationConfig(\n",
        "    data=config.datadir,\n",
        "    source_lang=config.source_lang,\n",
        "    target_lang=config.target_lang,\n",
        "    train_subset=\"train\",\n",
        "    required_seq_len_multiple=8,\n",
        "    dataset_impl=\"mmap\",\n",
        "    upsample_primary=1,\n",
        ")\n",
        "task = TranslationTask.setup_task(task_cfg)"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-08-03 02:31:04 | INFO | fairseq.tasks.translation | [en] dictionary: 8000 types\n",
            "2021-08-03 02:31:04 | INFO | fairseq.tasks.translation | [zh] dictionary: 8000 types\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MdKcECjFpP_C",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a8af1f88-500e-48fb-828b-a23aaed4234e"
      },
      "source": [
        "logger.info(\"loading data for epoch 1\")\n",
        "task.load_dataset(split=\"train\", epoch=1, combine=True) # combine if you have back-translation data.\n",
        "task.load_dataset(split=\"valid\", epoch=1)"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-08-03 02:31:15 | INFO | hw5.seq2seq | loading data for epoch 1\n",
            "2021-08-03 02:31:15 | INFO | fairseq.data.data_utils | loaded 390,041 examples from: ./DATA/data-bin/ted2020/train.en-zh.en\n",
            "2021-08-03 02:31:16 | INFO | fairseq.data.data_utils | loaded 390,041 examples from: ./DATA/data-bin/ted2020/train.en-zh.zh\n",
            "2021-08-03 02:31:16 | INFO | fairseq.tasks.translation | ./DATA/data-bin/ted2020 train en-zh 390041 examples\n",
            "2021-08-03 02:31:16 | INFO | fairseq.data.data_utils | loaded 3,939 examples from: ./DATA/data-bin/ted2020/valid.en-zh.en\n",
            "2021-08-03 02:31:16 | INFO | fairseq.data.data_utils | loaded 3,939 examples from: ./DATA/data-bin/ted2020/valid.en-zh.zh\n",
            "2021-08-03 02:31:16 | INFO | fairseq.tasks.translation | ./DATA/data-bin/ted2020 valid en-zh 3939 examples\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1vTtEqDipP_C",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "bc702a11-f504-444b-8ec4-be96873fc62f"
      },
      "source": [
        "sample = task.dataset(\"valid\")[1]\n",
        "pprint.pprint(sample)\n",
        "pprint.pprint(\n",
        "    \"Source: \" + \\\n",
        "    task.source_dictionary.string(\n",
        "        sample['source'],\n",
        "        config.post_process,\n",
        "    )\n",
        ")\n",
        "pprint.pprint(\n",
        "    \"Target: \" + \\\n",
        "    task.target_dictionary.string(\n",
        "        sample['target'],\n",
        "        config.post_process,\n",
        "    )\n",
        ")"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'id': 1,\n",
            " 'source': tensor([  18,   14,    6, 2234,   60,   19,   80,    5,  256,   16,  405, 1407,\n",
            "        1706,    7,    2]),\n",
            " 'target': tensor([ 140,  690,   28,  270,   45,  151, 1142,  660,  606,  369, 3114, 2434,\n",
            "        1434,  192,    2])}\n",
            "\"Source: that's exactly what i do optical mind control .\"\n",
            "'Target: 這實在就是我所做的--光學操控思想'\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ASiTDe6hpP_D"
      },
      "source": [
        "## Dataset Iterator"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9XuOfDj4pP_D"
      },
      "source": [
        "* 將每個 batch 控制在 N 個 token 讓 GPU 記憶體更有效被利用\n",
        "* 讓 training set 每個 epoch 有不同 shuffling\n",
        "* 濾掉長度太長的句子\n",
        "* 將每個 batch 內的句子 pad 成一樣長，好讓 GPU 平行運算\n",
        "* 加上 eos 並 shift 一格\n",
        "    - teacher forcing: 為了訓練模型根據prefix生成下個字，decoder的輸入會是輸出目標序列往右shift一格。\n",
        "    - 一般是會在輸入開頭加個bos token (如下圖)\n",
        "![seq2seq](https://i.imgur.com/0zeDyuI.png)\n",
        "    - fairseq 則是直接把 eos 挪到 beginning，訓練起來效果其實差不多。例如: \n",
        "    ```\n",
        "    # 輸出目標 (target) 和 Decoder輸入 (prev_output_tokens): \n",
        "                   eos = 2\n",
        "                target = 419,  711,  238,  888,  792,   60,  968,    8,    2\n",
        "    prev_output_tokens = 2,  419,  711,  238,  888,  792,   60,  968,    8\n",
        "    ```\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R4KbGqsPpP_D",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8833f5f5-cb24-4c1c-a6c2-734753e22e24"
      },
      "source": [
        "def load_data_iterator(task, split, epoch=1, max_tokens=4000, num_workers=1, cached=True):\n",
        "    batch_iterator = task.get_batch_iterator(\n",
        "        dataset=task.dataset(split),\n",
        "        max_tokens=max_tokens,\n",
        "        max_sentences=None,\n",
        "        max_positions=utils.resolve_max_positions(\n",
        "            task.max_positions(),\n",
        "            max_tokens,\n",
        "        ),\n",
        "        ignore_invalid_inputs=True,\n",
        "        seed=seed,\n",
        "        num_workers=num_workers,\n",
        "        epoch=epoch,\n",
        "        disable_iterator_cache=not cached,\n",
        "        # Set this to False to speed up. However, if set to False, changing max_tokens beyond \n",
        "        # first call of this method has no effect. \n",
        "    )\n",
        "    return batch_iterator\n",
        "\n",
        "demo_epoch_obj = load_data_iterator(task, \"valid\", epoch=1, max_tokens=20, num_workers=1, cached=False)\n",
        "demo_iter = demo_epoch_obj.next_epoch_itr(shuffle=True)\n",
        "sample = next(demo_iter)\n",
        "sample"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-08-03 02:31:37 | WARNING | fairseq.tasks.fairseq_task | 2,532 samples have invalid sizes and will be skipped, max_positions=(20, 20), first few sample ids=[29, 135, 2444, 3058, 682, 731, 235, 1558, 3383, 559]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'id': tensor([723]),\n",
              " 'net_input': {'prev_output_tokens': tensor([[   2,  140,  296,  318, 1560,   51,  568,  316,  225, 1952,  254,   78,\n",
              "            151, 2691,    9,  215, 1680,   10,    1,    1,    1,    1,    1,    1]]),\n",
              "  'src_lengths': tensor([19]),\n",
              "  'src_tokens': tensor([[   1,    1,    1,    1,    1,   18,   26,   82,    8,  480,   15,  651,\n",
              "           1361,   38,    6,  176, 2696,   39,    5,  822,   92,  260,    7,    2]])},\n",
              " 'nsentences': 1,\n",
              " 'ntokens': 18,\n",
              " 'target': tensor([[ 140,  296,  318, 1560,   51,  568,  316,  225, 1952,  254,   78,  151,\n",
              "          2691,    9,  215, 1680,   10,    2,    1,    1,    1,    1,    1,    1]])}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "N1YRZFb1pP_D"
      },
      "source": [
        "* 每個 batch 是一個字典，key 是字串，value 是 Tensor，內容說明如下\n",
        "```python\n",
        "batch = {\n",
        "    \"id\": id, # 每個 example 的 id\n",
        "    \"nsentences\": len(samples), # batch size 句子數\n",
        "    \"ntokens\": ntokens, # batch size 字數\n",
        "    \"net_input\": {\n",
        "        \"src_tokens\": src_tokens, # 來源語言的序列\n",
        "        \"src_lengths\": src_lengths, # 每句話沒有 pad 過的長度\n",
        "        \"prev_output_tokens\": prev_output_tokens, # 上面提到右 shift 一格後的目標序列\n",
        "    },\n",
        "    \"target\": target, # 目標序列\n",
        "}\n",
        "```"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vjfHpU9epP_E"
      },
      "source": [
        "# 定義模型架構\n",
        "* 我們一樣繼承 fairseq 的 encoder, decoder 和 model, 這樣測試階段才能直接用他寫好的 beam search 函式"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S-XGV9krpP_E"
      },
      "source": [
        "from fairseq.models import (\n",
        "    FairseqEncoder, \n",
        "    FairseqIncrementalDecoder,\n",
        "    FairseqEncoderDecoderModel\n",
        ")"
      ],
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ue-hgaOlpP_E"
      },
      "source": [
        "## Encoder 編碼器"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2w6le0K9pP_E"
      },
      "source": [
        "- seq2seq 模型的編碼器為 RNN 或 Transformer Encoder，以下說明以 RNN 為例，Transformer 略有不同。對於每個輸入，Encoder 會輸出一個向量和一個隱藏狀態(hidden state)，並將隱藏狀態用於下一個輸入。換句話說，Encoder 會逐步讀取輸入序列，並在每個 timestep 輸出單個向量，以及在最後 timestep 輸出最終隱藏狀態(content vector)\n",
        "- 參數:\n",
        "  - *args*\n",
        "      - encoder_embed_dim 是 embedding 的維度，主要將 one-hot vector 的單詞向量壓縮到指定的維度，主要是為了降維和濃縮資訊的功用\n",
        "      - encoder_ffn_embed_dim 是 RNN 輸出和隱藏狀態的維度(hidden dimension)\n",
        "      - encoder_layers 是 RNN 要疊多少層\n",
        "      - dropout 是決定有多少的機率會將某個節點變為 0，主要是為了防止 overfitting ，一般來說是在訓練時使用，測試時則不使用\n",
        "  - *dictionary*: fairseq 幫我們做好的 dictionary. 在此用來得到 padding index，好用來得到 encoder padding mask. \n",
        "  - *embed_tokens*: 事先做好的詞嵌入 (nn.Embedding)\n",
        "\n",
        "- 輸入: \n",
        "    - *src_tokens*: 英文的整數序列 e.g. 1, 28, 29, 205, 2 \n",
        "- 輸出: \n",
        "    - *outputs*: 最上層 RNN 每個 timestep 的輸出，後續可以用 Attention 再進行處理\n",
        "    - *final_hiddens*: 每層最終 timestep 的隱藏狀態，將傳遞到 Decoder 進行解碼\n",
        "    - *encoder_padding_mask*: 告訴我們哪些是位置的資訊不重要。\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rgnJIHwapP_E"
      },
      "source": [
        "class RNNEncoder(FairseqEncoder):\n",
        "    def __init__(self, args, dictionary, embed_tokens):\n",
        "        super().__init__(dictionary)\n",
        "        self.embed_tokens = embed_tokens\n",
        "        \n",
        "        self.embed_dim = args.encoder_embed_dim\n",
        "        self.hidden_dim = args.encoder_ffn_embed_dim\n",
        "        self.num_layers = args.encoder_layers\n",
        "        \n",
        "        self.dropout_in_module = nn.Dropout(args.dropout)\n",
        "        self.rnn = nn.GRU(\n",
        "            self.embed_dim, \n",
        "            self.hidden_dim, \n",
        "            self.num_layers, \n",
        "            dropout=args.dropout, \n",
        "            batch_first=False, \n",
        "            bidirectional=True\n",
        "        )\n",
        "        self.dropout_out_module = nn.Dropout(args.dropout)\n",
        "        \n",
        "        self.padding_idx = dictionary.pad()\n",
        "        \n",
        "    def combine_bidir(self, outs, bsz: int):\n",
        "        out = outs.view(self.num_layers, 2, bsz, -1).transpose(1, 2).contiguous()\n",
        "        return out.view(self.num_layers, bsz, -1)\n",
        "\n",
        "    def forward(self, src_tokens, **unused):\n",
        "        bsz, seqlen = src_tokens.size()\n",
        "        \n",
        "        # get embeddings\n",
        "        x = self.embed_tokens(src_tokens)\n",
        "        x = self.dropout_in_module(x)\n",
        "\n",
        "        # B x T x C -> T x B x C\n",
        "        x = x.transpose(0, 1)\n",
        "        \n",
        "        # 過雙向RNN\n",
        "        h0 = x.new_zeros(2 * self.num_layers, bsz, self.hidden_dim)\n",
        "        x, final_hiddens = self.rnn(x, h0)\n",
        "        outputs = self.dropout_out_module(x)\n",
        "        # outputs = [sequence len, batch size, hid dim * directions] 是最上層RNN的輸出\n",
        "        # hidden =  [num_layers * directions, batch size  , hid dim]\n",
        "        \n",
        "        # 因為 Encoder 是雙向的RNN，所以需要將同一層兩個方向的 hidden state 接在一起\n",
        "        final_hiddens = self.combine_bidir(final_hiddens, bsz)\n",
        "        # hidden =  [num_layers x batch x num_directions*hidden]\n",
        "        \n",
        "        encoder_padding_mask = src_tokens.eq(self.padding_idx).t()\n",
        "        return tuple(\n",
        "            (\n",
        "                outputs,  # seq_len x batch x hidden\n",
        "                final_hiddens,  # num_layers x batch x num_directions*hidden\n",
        "                encoder_padding_mask,  # seq_len x batch\n",
        "            )\n",
        "        )\n",
        "    \n",
        "    def reorder_encoder_out(self, encoder_out, new_order):\n",
        "        # 這個beam search時會用到，意義並不是很重要\n",
        "        return tuple(\n",
        "            (\n",
        "                encoder_out[0].index_select(1, new_order),\n",
        "                encoder_out[1].index_select(1, new_order),\n",
        "                encoder_out[2].index_select(1, new_order),\n",
        "            )\n",
        "        )"
      ],
      "execution_count": 101,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3vYwAMgXpP_F"
      },
      "source": [
        "## Attention"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Gm5ZzawipP_F"
      },
      "source": [
        "- 當輸入過長，或是單獨靠 “content vector” 無法取得整個輸入的意思時，用 Attention Mechanism 來提供 Decoder 更多的資訊\n",
        "- 根據現在 **Decoder embeddings** ，去計算在 **Encoder outputs** 中，那些與其有較高的關係，根據關係的數值來把 Encoder outputs 平均起來作為 **Decoder** RNN 的輸入 \n",
        "- 常見 Attention 的實作是用 Neural Network / Dot Product 來算 **query** (decoder embeddings) 和 **key** (Encoder outputs) 之間的關係，再對所有算出來的數值做 **softmax** 得到分佈，最後根據這個分佈對 **values** (Encoder outputs) 做 **weight sum**\n",
        "\n",
        "- 參數:\n",
        "  - *input_embed_dim*: key 的維度，應是 decoder 要做 attend 時的向量的維度\n",
        "  - *source_embed_dim*: query 的維度，應是要被 attend 的向量(encoder outputs)的維度\n",
        "  - *output_embed_dim*: value 的維度，應是做完 attention 後，下一層預期的向量維度\n",
        "\n",
        "- 輸入: \n",
        "    - *inputs*: 就是 key，要 attend 別人的向量\n",
        "    - *encoder_outputs*: 是 query/value，被 attend 的向量\n",
        "    - *encoder_padding_mask*: 告訴我們哪些是位置的資訊不重要。\n",
        "- 輸出: \n",
        "    - *output*: 做完 attention 後的 context vector\n",
        "    - *attention score*: attention 的分布\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TTFcocCDpP_F"
      },
      "source": [
        "class AttentionLayer(nn.Module):\n",
        "    def __init__(self, input_embed_dim, source_embed_dim, output_embed_dim, bias=False):\n",
        "        super().__init__()\n",
        "\n",
        "        self.input_proj = nn.Linear(input_embed_dim, source_embed_dim, bias=bias)\n",
        "        self.output_proj = nn.Linear(\n",
        "            input_embed_dim + source_embed_dim, output_embed_dim, bias=bias\n",
        "        )\n",
        "\n",
        "    def forward(self, inputs, encoder_outputs, encoder_padding_mask):\n",
        "        # inputs: T, B, dim\n",
        "        # encoder_outputs: S x B x dim\n",
        "        # padding mask:  S x B\n",
        "        \n",
        "        # convert all to batch first\n",
        "        inputs = inputs.transpose(1,0) # B, T, dim\n",
        "        encoder_outputs = encoder_outputs.transpose(1,0) # B, S, dim\n",
        "        encoder_padding_mask = encoder_padding_mask.transpose(1,0) # B, S\n",
        "        \n",
        "        # 投影到encoder_outputs的維度\n",
        "        x = self.input_proj(inputs)\n",
        "\n",
        "        # 計算attention\n",
        "        # (B, T, dim) x (B, dim, S) = (B, T, S)\n",
        "        attn_scores = torch.bmm(x, encoder_outputs.transpose(1,2))\n",
        "\n",
        "        # 擋住padding位置的attention\n",
        "        if encoder_padding_mask is not None:\n",
        "            # 利用broadcast  B, S -> (B, 1, S)\n",
        "            encoder_padding_mask = encoder_padding_mask.unsqueeze(1)\n",
        "            attn_scores = (\n",
        "                attn_scores.float()\n",
        "                .masked_fill_(encoder_padding_mask, float(\"-inf\"))\n",
        "                .type_as(attn_scores)\n",
        "            )  # FP16 support: cast to float and back\n",
        "\n",
        "        # 在source對應維度softmax\n",
        "        attn_scores = F.softmax(attn_scores, dim=-1)\n",
        "\n",
        "        # 形狀 (B, T, S) x (B, S, dim) = (B, T, dim) 加權平均\n",
        "        x = torch.bmm(attn_scores, encoder_outputs)\n",
        "\n",
        "        # (B, T, dim)\n",
        "        x = torch.cat((x, inputs), dim=-1)\n",
        "        x = torch.tanh(self.output_proj(x)) # concat + linear + tanh\n",
        "        \n",
        "        # 回復形狀 (B, T, dim) -> (T, B, dim)\n",
        "        return x.transpose(1,0), attn_scores"
      ],
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RTNnFqDypP_G"
      },
      "source": [
        "## Decoder 解碼器"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AKGfCiyXpP_G"
      },
      "source": [
        "* 解碼器的 hidden states 會用編碼器最終隱藏狀態來初始化(content vector)\n",
        "* 解碼器同時也根據目前 timestep 的輸入(也就是前幾個 timestep 的 output)，改變 hidden states，並輸出結果 \n",
        "* 如果加入 attention 可以使表現更好\n",
        "* 我們把 seq2seq 步驟寫在解碼器裡，好讓等等 Seq2Seq 這個型別可以通用 RNN 和 Transformer，而不用再改寫\n",
        "- 參數:\n",
        "  - *args*\n",
        "      - decoder_embed_dim 是解碼器 embedding 的維度，類同 encoder_embed_dim，\n",
        "      - decoder_ffn_embed_dim 是解碼器 RNN 的隱藏維度，類同 encoder_ffn_embed_dim\n",
        "      - decoder_layers 解碼器 RNN 的層數\n",
        "      - share_decoder_input_output_embed 通常 decoder 最後輸出的投影矩陣會和輸入 embedding 共用參數\n",
        "  - *dictionary*: fairseq 幫我們做好的 dictionary.\n",
        "  - *embed_tokens*: 事先做好的詞嵌入(nn.Embedding)\n",
        "- 輸入: \n",
        "    - *prev_output_tokens*: 英文的整數序列 e.g. 1, 28, 29, 205, 2 已經 shift 一格的 target\n",
        "    - *encoder_out*: 編碼器的輸出\n",
        "    - *incremental_state*: 這是測試階段為了加速，所以會記錄每個 timestep 的 hidden state 詳見 forward\n",
        "- 輸出: \n",
        "    - *outputs*: decoder 每個 timestep 的 logits，還沒經過 softmax 的分布\n",
        "    - *extra*: 沒用到"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8oLTkhv5pP_G"
      },
      "source": [
        "class RNNDecoder(FairseqIncrementalDecoder):\n",
        "    def __init__(self, args, dictionary, embed_tokens):\n",
        "        super().__init__(dictionary)\n",
        "        self.embed_tokens = embed_tokens\n",
        "        \n",
        "        assert args.decoder_layers == args.encoder_layers, f\"\"\"seq2seq rnn requires that encoder \n",
        "        and decoder have same layers of rnn. got: {args.encoder_layers, args.decoder_layers}\"\"\"\n",
        "        assert args.decoder_ffn_embed_dim == args.encoder_ffn_embed_dim*2, f\"\"\"seq2seq-rnn requires \n",
        "        that decoder hidden to be 2*encoder hidden dim. got: {args.decoder_ffn_embed_dim, args.encoder_ffn_embed_dim*2}\"\"\"\n",
        "        \n",
        "        self.embed_dim = args.decoder_embed_dim\n",
        "        self.hidden_dim = args.decoder_ffn_embed_dim\n",
        "        self.num_layers = args.decoder_layers\n",
        "        \n",
        "        \n",
        "        self.dropout_in_module = nn.Dropout(args.dropout)\n",
        "        self.rnn = nn.GRU(\n",
        "            self.embed_dim, \n",
        "            self.hidden_dim, \n",
        "            self.num_layers, \n",
        "            dropout=args.dropout, \n",
        "            batch_first=False, \n",
        "            bidirectional=False\n",
        "        )\n",
        "        self.attention = AttentionLayer(\n",
        "            self.embed_dim, self.hidden_dim, self.embed_dim, bias=False\n",
        "        ) \n",
        "        # self.attention = None\n",
        "        self.dropout_out_module = nn.Dropout(args.dropout)\n",
        "        \n",
        "        if self.hidden_dim != self.embed_dim:\n",
        "            self.project_out_dim = nn.Linear(self.hidden_dim, self.embed_dim)\n",
        "        else:\n",
        "            self.project_out_dim = None\n",
        "        \n",
        "        if args.share_decoder_input_output_embed:\n",
        "            self.output_projection = nn.Linear(\n",
        "                self.embed_tokens.weight.shape[1],\n",
        "                self.embed_tokens.weight.shape[0],\n",
        "                bias=False,\n",
        "            )\n",
        "            self.output_projection.weight = self.embed_tokens.weight\n",
        "        else:\n",
        "            self.output_projection = nn.Linear(\n",
        "                self.output_embed_dim, len(dictionary), bias=False\n",
        "            )\n",
        "            nn.init.normal_(\n",
        "                self.output_projection.weight, mean=0, std=self.output_embed_dim ** -0.5\n",
        "            )\n",
        "        \n",
        "    def forward(self, prev_output_tokens, encoder_out, incremental_state=None, **unused):\n",
        "        # 取出encoder的輸出\n",
        "        encoder_outputs, encoder_hiddens, encoder_padding_mask = encoder_out\n",
        "        # outputs:          seq_len x batch x num_directions*hidden\n",
        "        # encoder_hiddens:  num_layers x batch x num_directions*encoder_hidden\n",
        "        # padding_mask:     seq_len x batch\n",
        "        \n",
        "        if incremental_state is not None and len(incremental_state) > 0:\n",
        "            # 有上個timestep留下的資訊，讀進來就可以繼續decode，不用從bos重來\n",
        "            prev_output_tokens = prev_output_tokens[:, -1:]\n",
        "            cache_state = self.get_incremental_state(incremental_state, \"cached_state\")\n",
        "            prev_hiddens = cache_state[\"prev_hiddens\"]\n",
        "        else:\n",
        "            # 沒有incremental state代表這是training或者是test time時的第一步\n",
        "            # 準備seq2seq: 把encoder_hiddens pass進去decoder的hidden states\n",
        "            prev_hiddens = encoder_hiddens\n",
        "        \n",
        "        bsz, seqlen = prev_output_tokens.size()\n",
        "        \n",
        "        # embed tokens\n",
        "        x = self.embed_tokens(prev_output_tokens)\n",
        "        x = self.dropout_in_module(x)\n",
        "\n",
        "        # B x T x C -> T x B x C\n",
        "        x = x.transpose(0, 1)\n",
        "                \n",
        "        # 做decoder-to-encoder attention\n",
        "        if self.attention is not None:\n",
        "            x, attn = self.attention(x, encoder_outputs, encoder_padding_mask)\n",
        "                        \n",
        "        # 過單向RNN\n",
        "        x, final_hiddens = self.rnn(x, prev_hiddens)\n",
        "        # outputs = [sequence len, batch size, hid dim]\n",
        "        # hidden =  [num_layers * directions, batch size  , hid dim]\n",
        "        x = self.dropout_out_module(x)\n",
        "                \n",
        "        # 投影到embedding size (如果hidden 和embed size不一樣，然後share_embedding又設成True,需要額外project一次)\n",
        "        if self.project_out_dim != None:\n",
        "            x = self.project_out_dim(x)\n",
        "        \n",
        "        # 投影到vocab size 的分佈\n",
        "        x = self.output_projection(x)\n",
        "        \n",
        "        # T x B x C -> B x T x C\n",
        "        x = x.transpose(1, 0)\n",
        "        \n",
        "        # 如果是Incremental, 記錄這個timestep的hidden states, 下個timestep讀回來\n",
        "        cache_state = {\n",
        "            \"prev_hiddens\": final_hiddens,\n",
        "        }\n",
        "        self.set_incremental_state(incremental_state, \"cached_state\", cache_state)\n",
        "        \n",
        "        return x, None\n",
        "    \n",
        "    def reorder_incremental_state(\n",
        "        self,\n",
        "        incremental_state,\n",
        "        new_order,\n",
        "    ):\n",
        "        # 這個beam search時會用到，意義並不是很重要\n",
        "        cache_state = self.get_incremental_state(incremental_state, \"cached_state\")\n",
        "        prev_hiddens = cache_state[\"prev_hiddens\"]\n",
        "        prev_hiddens = [p.index_select(0, new_order) for p in prev_hiddens]\n",
        "        cache_state = {\n",
        "            \"prev_hiddens\": torch.stack(prev_hiddens),\n",
        "        }\n",
        "        self.set_incremental_state(incremental_state, \"cached_state\", cache_state)\n",
        "        return"
      ],
      "execution_count": 103,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8fd1D5mgpP_H"
      },
      "source": [
        "## Seq2Seq\n",
        "- 由 **Encoder** 和 **Decoder** 組成\n",
        "- 接收輸入並傳給 **Encoder** \n",
        "- 將 **Encoder** 的輸出傳給 **Decoder**\n",
        "- **Decoder** 根據前幾個 timestep 的輸出和 **Encoder** 輸出進行解碼  \n",
        "- 當解碼完成後，將 **Decoder** 的輸出傳回 "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HmEDx7D-pP_H"
      },
      "source": [
        "class Seq2Seq(FairseqEncoderDecoderModel):\n",
        "    def __init__(self, args, encoder, decoder):\n",
        "        super().__init__(encoder, decoder)\n",
        "        self.args = args\n",
        "    \n",
        "    def forward(\n",
        "        self,\n",
        "        src_tokens,\n",
        "        src_lengths,\n",
        "        prev_output_tokens,\n",
        "        return_all_hiddens: bool = True,\n",
        "    ):\n",
        "        \"\"\"\n",
        "        Run the forward pass for an encoder-decoder model.\n",
        "        \"\"\"\n",
        "        encoder_out = self.encoder(\n",
        "            src_tokens, src_lengths=src_lengths, return_all_hiddens=return_all_hiddens\n",
        "        )\n",
        "        logits, extra = self.decoder(\n",
        "            prev_output_tokens,\n",
        "            encoder_out=encoder_out,\n",
        "            src_lengths=src_lengths,\n",
        "            return_all_hiddens=return_all_hiddens,\n",
        "        )\n",
        "        return logits, extra"
      ],
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "soGFoIiypP_H"
      },
      "source": [
        "# 模型初始化"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1SSMrK92pP_H"
      },
      "source": [
        "# HINT: transformer 架構\n",
        "from fairseq.models.transformer import (\n",
        "    TransformerEncoder, \n",
        "    TransformerDecoder,\n",
        ")\n",
        "\n",
        "def build_model(args, task):\n",
        "    \"\"\" 按照參數設定建置模型 \"\"\"\n",
        "    src_dict, tgt_dict = task.source_dictionary, task.target_dictionary\n",
        "\n",
        "    # 詞嵌入\n",
        "    encoder_embed_tokens = nn.Embedding(len(src_dict), args.encoder_embed_dim, src_dict.pad())\n",
        "    decoder_embed_tokens = nn.Embedding(len(tgt_dict), args.decoder_embed_dim, tgt_dict.pad())\n",
        "    \n",
        "    # 編碼器與解碼器\n",
        "    # TODO: 替換成 TransformerEncoder 和 TransformerDecoder\n",
        "    encoder = TransformerEncoder(args, src_dict, encoder_embed_tokens)\n",
        "    decoder = TransformerDecoder(args, tgt_dict, decoder_embed_tokens)\n",
        "    \n",
        "    # 序列到序列模型\n",
        "    model = Seq2Seq(args, encoder, decoder)\n",
        "    \n",
        "    # 序列到序列模型的初始化很重要 需要特別處理\n",
        "    def init_params(module):\n",
        "        from fairseq.modules import MultiheadAttention\n",
        "        if isinstance(module, nn.Linear):\n",
        "            module.weight.data.normal_(mean=0.0, std=0.02)\n",
        "            if module.bias is not None:\n",
        "                module.bias.data.zero_()\n",
        "        if isinstance(module, nn.Embedding):\n",
        "            module.weight.data.normal_(mean=0.0, std=0.02)\n",
        "            if module.padding_idx is not None:\n",
        "                module.weight.data[module.padding_idx].zero_()\n",
        "        if isinstance(module, MultiheadAttention):\n",
        "            module.q_proj.weight.data.normal_(mean=0.0, std=0.02)\n",
        "            module.k_proj.weight.data.normal_(mean=0.0, std=0.02)\n",
        "            module.v_proj.weight.data.normal_(mean=0.0, std=0.02)\n",
        "        if isinstance(module, nn.RNNBase):\n",
        "            for name, param in module.named_parameters():\n",
        "                if \"weight\" in name or \"bias\" in name:\n",
        "                    param.data.uniform_(-0.1, 0.1)\n",
        "            \n",
        "    # 初始化模型\n",
        "    model.apply(init_params)\n",
        "    return model"
      ],
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OCnIeyanpP_I"
      },
      "source": [
        "## 設定模型相關參數\n",
        "參考參數\n",
        "\n",
        "|model|embedding dim|encoder ffn|encoder layers|decoder ffn|decoder layers|\n",
        "|-|-|-|-|-|-|\n",
        "|RNN|256|512|1|1024|1|\n",
        "|Transformer|256|1024|4|1024|4|\n",
        "\n",
        "Strong baseline 用的參數可以參考 [Attention is all you need](#vaswani2017) 的 Table 3 的 transformer-base"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uDFkwQl4pP_I"
      },
      "source": [
        "arch_args = Namespace(\n",
        "    encoder_embed_dim=256,\n",
        "    encoder_ffn_embed_dim=1024,\n",
        "    encoder_layers=4,\n",
        "    decoder_embed_dim=256,\n",
        "    decoder_ffn_embed_dim=1024,\n",
        "    decoder_layers=4,\n",
        "    share_decoder_input_output_embed=True,\n",
        "    dropout=0.3,\n",
        ")\n",
        "\n",
        "# HINT: 補上Transformer用的參數\n",
        "def add_transformer_args(args):\n",
        "    args.encoder_attention_heads=4\n",
        "    args.encoder_normalize_before=True\n",
        "    \n",
        "    args.decoder_attention_heads=4\n",
        "    args.decoder_normalize_before=True\n",
        "    \n",
        "    args.activation_fn=\"relu\"\n",
        "    args.max_source_positions=1024\n",
        "    args.max_target_positions=1024\n",
        "    \n",
        "    # 補上我們沒有設定的Transformer預設參數\n",
        "    from fairseq.models.transformer import base_architecture \n",
        "    base_architecture(arch_args)\n",
        "\n",
        "add_transformer_args(arch_args)"
      ],
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gmMR3nPypP_I"
      },
      "source": [
        "if config.use_wandb:\n",
        "    wandb.config.update(vars(arch_args))"
      ],
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vvFKTX2zpP_I",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3ac2551d-d962-4340-8b44-8c2e3dd08321"
      },
      "source": [
        "model = build_model(arch_args, task)\n",
        "logger.info(model)"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-08-03 02:33:02 | INFO | hw5.seq2seq | Seq2Seq(\n",
            "  (encoder): TransformerEncoder(\n",
            "    (dropout_module): FairseqDropout()\n",
            "    (embed_tokens): Embedding(8000, 256, padding_idx=1)\n",
            "    (embed_positions): SinusoidalPositionalEmbedding()\n",
            "    (layers): ModuleList(\n",
            "      (0): TransformerEncoderLayer(\n",
            "        (self_attn): MultiheadAttention(\n",
            "          (dropout_module): FairseqDropout()\n",
            "          (k_proj): Linear(in_features=256, out_features=256, bias=True)\n",
            "          (v_proj): Linear(in_features=256, out_features=256, bias=True)\n",
            "          (q_proj): Linear(in_features=256, out_features=256, bias=True)\n",
            "          (out_proj): Linear(in_features=256, out_features=256, bias=True)\n",
            "        )\n",
            "        (self_attn_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
            "        (dropout_module): FairseqDropout()\n",
            "        (activation_dropout_module): FairseqDropout()\n",
            "        (fc1): Linear(in_features=256, out_features=1024, bias=True)\n",
            "        (fc2): Linear(in_features=1024, out_features=256, bias=True)\n",
            "        (final_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
            "      )\n",
            "      (1): TransformerEncoderLayer(\n",
            "        (self_attn): MultiheadAttention(\n",
            "          (dropout_module): FairseqDropout()\n",
            "          (k_proj): Linear(in_features=256, out_features=256, bias=True)\n",
            "          (v_proj): Linear(in_features=256, out_features=256, bias=True)\n",
            "          (q_proj): Linear(in_features=256, out_features=256, bias=True)\n",
            "          (out_proj): Linear(in_features=256, out_features=256, bias=True)\n",
            "        )\n",
            "        (self_attn_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
            "        (dropout_module): FairseqDropout()\n",
            "        (activation_dropout_module): FairseqDropout()\n",
            "        (fc1): Linear(in_features=256, out_features=1024, bias=True)\n",
            "        (fc2): Linear(in_features=1024, out_features=256, bias=True)\n",
            "        (final_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
            "      )\n",
            "      (2): TransformerEncoderLayer(\n",
            "        (self_attn): MultiheadAttention(\n",
            "          (dropout_module): FairseqDropout()\n",
            "          (k_proj): Linear(in_features=256, out_features=256, bias=True)\n",
            "          (v_proj): Linear(in_features=256, out_features=256, bias=True)\n",
            "          (q_proj): Linear(in_features=256, out_features=256, bias=True)\n",
            "          (out_proj): Linear(in_features=256, out_features=256, bias=True)\n",
            "        )\n",
            "        (self_attn_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
            "        (dropout_module): FairseqDropout()\n",
            "        (activation_dropout_module): FairseqDropout()\n",
            "        (fc1): Linear(in_features=256, out_features=1024, bias=True)\n",
            "        (fc2): Linear(in_features=1024, out_features=256, bias=True)\n",
            "        (final_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
            "      )\n",
            "      (3): TransformerEncoderLayer(\n",
            "        (self_attn): MultiheadAttention(\n",
            "          (dropout_module): FairseqDropout()\n",
            "          (k_proj): Linear(in_features=256, out_features=256, bias=True)\n",
            "          (v_proj): Linear(in_features=256, out_features=256, bias=True)\n",
            "          (q_proj): Linear(in_features=256, out_features=256, bias=True)\n",
            "          (out_proj): Linear(in_features=256, out_features=256, bias=True)\n",
            "        )\n",
            "        (self_attn_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
            "        (dropout_module): FairseqDropout()\n",
            "        (activation_dropout_module): FairseqDropout()\n",
            "        (fc1): Linear(in_features=256, out_features=1024, bias=True)\n",
            "        (fc2): Linear(in_features=1024, out_features=256, bias=True)\n",
            "        (final_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
            "      )\n",
            "    )\n",
            "    (layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
            "  )\n",
            "  (decoder): TransformerDecoder(\n",
            "    (dropout_module): FairseqDropout()\n",
            "    (embed_tokens): Embedding(8000, 256, padding_idx=1)\n",
            "    (embed_positions): SinusoidalPositionalEmbedding()\n",
            "    (layers): ModuleList(\n",
            "      (0): TransformerDecoderLayer(\n",
            "        (dropout_module): FairseqDropout()\n",
            "        (self_attn): MultiheadAttention(\n",
            "          (dropout_module): FairseqDropout()\n",
            "          (k_proj): Linear(in_features=256, out_features=256, bias=True)\n",
            "          (v_proj): Linear(in_features=256, out_features=256, bias=True)\n",
            "          (q_proj): Linear(in_features=256, out_features=256, bias=True)\n",
            "          (out_proj): Linear(in_features=256, out_features=256, bias=True)\n",
            "        )\n",
            "        (activation_dropout_module): FairseqDropout()\n",
            "        (self_attn_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
            "        (encoder_attn): MultiheadAttention(\n",
            "          (dropout_module): FairseqDropout()\n",
            "          (k_proj): Linear(in_features=256, out_features=256, bias=True)\n",
            "          (v_proj): Linear(in_features=256, out_features=256, bias=True)\n",
            "          (q_proj): Linear(in_features=256, out_features=256, bias=True)\n",
            "          (out_proj): Linear(in_features=256, out_features=256, bias=True)\n",
            "        )\n",
            "        (encoder_attn_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
            "        (fc1): Linear(in_features=256, out_features=1024, bias=True)\n",
            "        (fc2): Linear(in_features=1024, out_features=256, bias=True)\n",
            "        (final_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
            "      )\n",
            "      (1): TransformerDecoderLayer(\n",
            "        (dropout_module): FairseqDropout()\n",
            "        (self_attn): MultiheadAttention(\n",
            "          (dropout_module): FairseqDropout()\n",
            "          (k_proj): Linear(in_features=256, out_features=256, bias=True)\n",
            "          (v_proj): Linear(in_features=256, out_features=256, bias=True)\n",
            "          (q_proj): Linear(in_features=256, out_features=256, bias=True)\n",
            "          (out_proj): Linear(in_features=256, out_features=256, bias=True)\n",
            "        )\n",
            "        (activation_dropout_module): FairseqDropout()\n",
            "        (self_attn_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
            "        (encoder_attn): MultiheadAttention(\n",
            "          (dropout_module): FairseqDropout()\n",
            "          (k_proj): Linear(in_features=256, out_features=256, bias=True)\n",
            "          (v_proj): Linear(in_features=256, out_features=256, bias=True)\n",
            "          (q_proj): Linear(in_features=256, out_features=256, bias=True)\n",
            "          (out_proj): Linear(in_features=256, out_features=256, bias=True)\n",
            "        )\n",
            "        (encoder_attn_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
            "        (fc1): Linear(in_features=256, out_features=1024, bias=True)\n",
            "        (fc2): Linear(in_features=1024, out_features=256, bias=True)\n",
            "        (final_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
            "      )\n",
            "      (2): TransformerDecoderLayer(\n",
            "        (dropout_module): FairseqDropout()\n",
            "        (self_attn): MultiheadAttention(\n",
            "          (dropout_module): FairseqDropout()\n",
            "          (k_proj): Linear(in_features=256, out_features=256, bias=True)\n",
            "          (v_proj): Linear(in_features=256, out_features=256, bias=True)\n",
            "          (q_proj): Linear(in_features=256, out_features=256, bias=True)\n",
            "          (out_proj): Linear(in_features=256, out_features=256, bias=True)\n",
            "        )\n",
            "        (activation_dropout_module): FairseqDropout()\n",
            "        (self_attn_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
            "        (encoder_attn): MultiheadAttention(\n",
            "          (dropout_module): FairseqDropout()\n",
            "          (k_proj): Linear(in_features=256, out_features=256, bias=True)\n",
            "          (v_proj): Linear(in_features=256, out_features=256, bias=True)\n",
            "          (q_proj): Linear(in_features=256, out_features=256, bias=True)\n",
            "          (out_proj): Linear(in_features=256, out_features=256, bias=True)\n",
            "        )\n",
            "        (encoder_attn_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
            "        (fc1): Linear(in_features=256, out_features=1024, bias=True)\n",
            "        (fc2): Linear(in_features=1024, out_features=256, bias=True)\n",
            "        (final_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
            "      )\n",
            "      (3): TransformerDecoderLayer(\n",
            "        (dropout_module): FairseqDropout()\n",
            "        (self_attn): MultiheadAttention(\n",
            "          (dropout_module): FairseqDropout()\n",
            "          (k_proj): Linear(in_features=256, out_features=256, bias=True)\n",
            "          (v_proj): Linear(in_features=256, out_features=256, bias=True)\n",
            "          (q_proj): Linear(in_features=256, out_features=256, bias=True)\n",
            "          (out_proj): Linear(in_features=256, out_features=256, bias=True)\n",
            "        )\n",
            "        (activation_dropout_module): FairseqDropout()\n",
            "        (self_attn_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
            "        (encoder_attn): MultiheadAttention(\n",
            "          (dropout_module): FairseqDropout()\n",
            "          (k_proj): Linear(in_features=256, out_features=256, bias=True)\n",
            "          (v_proj): Linear(in_features=256, out_features=256, bias=True)\n",
            "          (q_proj): Linear(in_features=256, out_features=256, bias=True)\n",
            "          (out_proj): Linear(in_features=256, out_features=256, bias=True)\n",
            "        )\n",
            "        (encoder_attn_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
            "        (fc1): Linear(in_features=256, out_features=1024, bias=True)\n",
            "        (fc2): Linear(in_features=1024, out_features=256, bias=True)\n",
            "        (final_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
            "      )\n",
            "    )\n",
            "    (layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
            "    (output_projection): Linear(in_features=256, out_features=8000, bias=False)\n",
            "  )\n",
            ")\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "m7NHktqKpP_J"
      },
      "source": [
        "# Optimization 最佳化"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "N9uAre01pP_J"
      },
      "source": [
        "## Loss: Label Smoothing Regularization\n",
        "* 讓模型學習輸出較不集中的分佈，防止模型過度自信\n",
        "* 有時候Ground Truth並非唯一答案，所以在算loss時，我們會保留一部份機率給正確答案以外的label\n",
        "* 可以有效防止過度擬合\n",
        "\n",
        "code [source](https://fairseq.readthedocs.io/en/latest/_modules/fairseq/criterions/label_smoothed_cross_entropy.html)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2Ooo0T38pP_J"
      },
      "source": [
        "class LabelSmoothedCrossEntropyCriterion(nn.Module):\n",
        "    def __init__(self, smoothing, ignore_index=None, reduce=True):\n",
        "        super().__init__()\n",
        "        self.smoothing = smoothing\n",
        "        self.ignore_index = ignore_index\n",
        "        self.reduce = reduce\n",
        "    \n",
        "    def forward(self, lprobs, target):\n",
        "        if target.dim() == lprobs.dim() - 1:\n",
        "            target = target.unsqueeze(-1)\n",
        "        # nll: Negative log likelihood，當目標是one-hot時的cross-entropy loss. 以下同 F.nll_loss\n",
        "        nll_loss = -lprobs.gather(dim=-1, index=target)\n",
        "        # 將一部分正確答案的機率分配給其他label 所以當計算cross-entropy時等於把所有label的log prob加起來\n",
        "        smooth_loss = -lprobs.sum(dim=-1, keepdim=True)\n",
        "        if self.ignore_index is not None:\n",
        "            pad_mask = target.eq(self.ignore_index)\n",
        "            nll_loss.masked_fill_(pad_mask, 0.0)\n",
        "            smooth_loss.masked_fill_(pad_mask, 0.0)\n",
        "        else:\n",
        "            nll_loss = nll_loss.squeeze(-1)\n",
        "            smooth_loss = smooth_loss.squeeze(-1)\n",
        "        if self.reduce:\n",
        "            nll_loss = nll_loss.sum()\n",
        "            smooth_loss = smooth_loss.sum()\n",
        "        # 計算cross-entropy時 加入分配給其他label的loss\n",
        "        eps_i = self.smoothing / lprobs.size(-1)\n",
        "        loss = (1.0 - self.smoothing) * nll_loss + eps_i * smooth_loss\n",
        "        return loss\n",
        "\n",
        "# 一般都用0.1效果就很好了\n",
        "criterion = LabelSmoothedCrossEntropyCriterion(\n",
        "    smoothing=0.1,\n",
        "    ignore_index=task.target_dictionary.pad(),\n",
        ")"
      ],
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TmNqHZAbpP_J"
      },
      "source": [
        "## Optimizer: Adam + lr scheduling\n",
        "Inverse square root 排程對於訓練 Transformer 時的穩定性很重要，後來也用在 RNN 上。\n",
        "根據底下公式來更新 learning rate，前期線性增長，後期根據更新步數方根的倒數來遞減。\n",
        "$$lrate = d_{\\text{model}}^{-0.5}\\cdot\\min({step\\_num}^{-0.5},{step\\_num}\\cdot{warmup\\_steps}^{-1.5})$$\n",
        "code [source](https://nlp.seas.harvard.edu/2018/04/03/attention.html)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-aO0wh75pP_K"
      },
      "source": [
        "class NoamOpt:\n",
        "    \"Optim wrapper that implements rate.\"\n",
        "    def __init__(self, model_size, factor, warmup, optimizer):\n",
        "        self.optimizer = optimizer\n",
        "        self._step = 0\n",
        "        self.warmup = warmup\n",
        "        self.factor = factor\n",
        "        self.model_size = model_size\n",
        "        self._rate = 0\n",
        "    \n",
        "    @property\n",
        "    def param_groups(self):\n",
        "        return self.optimizer.param_groups\n",
        "        \n",
        "    def multiply_grads(self, c):\n",
        "        \"\"\"Multiplies grads by a constant *c*.\"\"\"                \n",
        "        for group in self.param_groups:\n",
        "            for p in group['params']:\n",
        "                if p.grad is not None:\n",
        "                    p.grad.data.mul_(c)\n",
        "        \n",
        "    def step(self):\n",
        "        \"Update parameters and rate\"\n",
        "        self._step += 1\n",
        "        rate = self.rate()\n",
        "        for p in self.param_groups:\n",
        "            p['lr'] = rate\n",
        "        self._rate = rate\n",
        "        self.optimizer.step()\n",
        "        \n",
        "    def rate(self, step = None):\n",
        "        \"Implement `lrate` above\"\n",
        "        if step is None:\n",
        "            step = self._step\n",
        "        return 0 if not step else self.factor * \\\n",
        "            (self.model_size ** (-0.5) *\n",
        "            min(step ** (-0.5), step * self.warmup ** (-1.5)))"
      ],
      "execution_count": 34,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iZHg8sCipP_K"
      },
      "source": [
        "## 排程視覺化"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S2r8d2oNpP_K",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 265
        },
        "outputId": "968b5c7d-08bc-4606-eca9-e228b81f35ed"
      },
      "source": [
        "optimizer = NoamOpt(\n",
        "    model_size=arch_args.encoder_embed_dim, \n",
        "    factor=config.lr_factor, \n",
        "    warmup=config.lr_warmup, \n",
        "    optimizer=torch.optim.AdamW(model.parameters(), lr=0, betas=(0.9, 0.98), eps=1e-9, weight_decay=0.0001))\n",
        "plt.plot(np.arange(1, 100000), [optimizer.rate(i) for i in range(1, 100000)])\n",
        "plt.legend([f\"{optimizer.model_size}:{optimizer.warmup}\"])\n",
        "None"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZAAAAD4CAYAAADCb7BPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de3xV1Znw8d+Te0JuEAKEBEiQYAmgoBHxnWptqYLYEae1LYytOkPrtNWP09p5rcy8Yzu09q11OlVHrbVVi44Vea2X1FKx3i9VMFy8AKLhJoFAQoCcBHI5J3neP/ZKOIRzkpPrSQ7P9/M5n+yz9tprr50D58laa++1RFUxxhhjeiou2hUwxhgzPFkAMcYY0ysWQIwxxvSKBRBjjDG9YgHEGGNMryREuwIDafTo0VpYWBjtahhjzLCyfv36g6qa212+mA4ghYWFlJeXR7saxhgzrIjI7kjyWReWMcaYXrEAYowxplcsgBhjjOmViMZARGQBcCcQD/xWVX/WaX8y8DBwNlALfFVVd7l9y4ClQCtwg6quEZEJLv9YQIH7VfVOl38U8DhQCOwCvqKqh0VEXB0WAseAa1R1Q6+v3BgzbPn9fiorK2lqaop2VYa1lJQUCgoKSExM7NXx3QYQEYkH7gEuAiqBd0SkTFW3BGVbChxW1Skishi4DfiqiJQAi4HpwHjgBRGZCgSA76vqBhHJANaLyF9cmTcDL6rqz0TkZvf+B8AlQLF7nQv8yv00xpxiKisrycjIoLCwEO9vS9NTqkptbS2VlZUUFRX1qoxIurDmABWqukNVW4CVwKJOeRYBK9z2E8A812JYBKxU1WZV3QlUAHNUtaq99aCq9cBWID9EWSuAy4PSH1bP20C2iOT18HqNMTGgqamJnJwcCx59ICLk5OT0qRUXSQDJB/YEva/k+Jf9SXlUNQDUATmRHCsihcBsYK1LGquqVW57P143V6T1QESuFZFyESmvqanp/uqMMcOSBY++6+vvMKqD6CKSDvwB+K6q+jrvV2+u+R7NN6+q96tqqaqW5uZ2+xxMv1n9fhUHG5oH7XzGGBNtkQSQvcCEoPcFLi1kHhFJALLwBtPDHisiiXjB41FVfTIoz4H2rin3s7oH9YiK+iY/33l0A1c9sC7aVTHGDII9e/bw2c9+lpKSEqZPn86dd94JwI9+9CPy8/OZNWsWs2bNYvXq1R3HvPfee5x33nlMnz6dmTNndtl19Itf/AIR4eDBg4A3XnHDDTcwZcoUzjjjDDZsOH7/0IoVKyguLqa4uJgVK1Z0pK9fv56ZM2cyZcoUbrjhBgZk7SdV7fKFN9C+AygCkoB3gemd8lwH3Oe2FwOr3PZ0lz/ZHb8D704uwbsL644Q57sduNlt3wz83G1fCvzZHTsXWNdd3c8++2wdDJWHj+mkHzyrk37w7KCcz5hT3ZYtW6J6/n379un69etVVdXn82lxcbFu3rxZf/jDH+rtt99+Un6/368zZ87UTZs2qarqwYMHNRAIhCz7k08+0YsvvlgnTpyoNTU1qqr6pz/9SRcsWKBtbW361ltv6Zw5c1RVtba2VouKirS2tlYPHTqkRUVFeujQIVVVPeecc/Stt97StrY2XbBgga5evTrk+UL9LoFy7eb7VVW7b4GoN6ZxPbAGb7B7lapuFpHlInKZy/YAkCMiFcCN7osfVd0MrAK2AM8B16lqK/A3wNeBz4nIJvda6Mr6GXCRiHwMfN69B1jtAlAF8BvgO93VfbD4Gv3RroIxZhDl5eVx1llnAZCRkcG0adPYuzd8h8jzzz/PGWecwZlnnglATk4O8fHxIfN+73vf4+c///kJ4xPPPPMMV111FSLC3LlzOXLkCFVVVaxZs4aLLrqIUaNGMXLkSC666CKee+45qqqq8Pl8zJ07FxHhqquu4umnn+7H34AnoudAVHU13hd4cNotQdtNwJfDHHsrcGuntDfwWhKh8tcC80KkK15LZ8ixAGJM9PzHHzezZd9JQ6h9UjI+kx/+7fSI8u7atYuNGzdy7rnn8uabb3L33Xfz8MMPU1payi9+8QtGjhzJRx99hIgwf/58ampqWLx4MTfddBMA3/jGN/jWt75FaWkpzzzzDPn5+R2Bpt3evXuZMOF4D35BQQF79+7tMr2goOCk9P5mT6L3g/qmQMf2sZZAFzmNMbGkoaGBL33pS9xxxx1kZmby7W9/m+3bt7Np0yby8vL4/ve/D0AgEOCNN97g0Ucf5Y033uCpp57ixRdfBOC3v/0tpaWlHDt2jJ/+9KcsX748mpfUIzE9G+9g8TUdb4HsOdTI6eMyolgbY04tkbYU+pvf7+dLX/oSV155JV/84hcBGDt2bMf+b37zm3zhC18AvBbABRdcwOjRowFYuHAhGzZsYN68450t27dvZ+fOnR2tj8rKSs466yzWrVtHfn4+e/Ycf4qhsrKS/Px88vPzeeWVV05Iv/DCC8nPz6eysvKk/P3NWiD9ILgLa3ft0SjWxBgzGFSVpUuXMm3aNG688caO9Kqqqo7tp556ihkzZgAwf/583n//fY4dO0YgEODVV1+lpKTkhDJnzpxJdXU1u3btYteuXRQUFLBhwwbGjRvHZZddxsMPP4yq8vbbb5OVlUVeXh7z58/n+eef5/Dhwxw+fJjnn3+e+fPnk5eXR2ZmJm+//TaqysMPP8yiRZ2f/+47a4H0A19QF9Ynh45FsSbGmMHw5ptv8sgjjzBz5kxmzZoFwE9/+lMee+wxNm3ahIhQWFjIr3/9awBGjhzJjTfeyDnnnIOIsHDhQi699FLgxDGQcBYuXMjq1auZMmUKaWlpPPTQQwCMGjWKf//3f+ecc84B4JZbbmHUqFEA3HvvvVxzzTU0NjZyySWXcMkll/T770F0IO4NHiJKS0t1MBaUuvVPW3jk7d0kxsfxd7PzWb5oxoCf05hT2datW5k2bVq0qxETQv0uRWS9qoaPaI61QPqBrzFAVmoio9OT2V1rLRBjzKnBxkD6ga/JT0ZKIpNy0thjXVjGmFOEBZB+4Gvyk5mSwMRRI9hz+BiB1rZoV8mYmBfL3e+Dpa+/Qwsg/aC+KUBmaiKn5Y7A36o2kG7MAEtJSaG2ttaCSB+oWw8kJSWl12XYGEg/8DX6KcwZwZQx6QBsrznK5Nz0KNfKmNhVUFBAZWUltmRD37SvSNhbFkD6ga8pQEZKAqe5AFJR3cBFJWO7OcoY01uJiYm9XkXP9B/rwuojVcXX6CczNZHMlETGZCRTUd0Q7WoZY8yAswDSR43+VgJtSmaKtyj9abnpbK+xAGKMiX0WQPqofSLFzFSvN3DKmHS2VzfY4J4xJuZZAOmj9nmwMjpaICOobw5QU2/L2xpjYpsFkD5qn4k3M6W9BeLNxGvjIMaYWBdRABGRBSKyTUQqROTmEPuTReRxt3+tiBQG7Vvm0reJyPyg9AdFpFpEPuhU1uNBqxTuEpFNLr1QRBqD9t3X24vuT77G9i4s1wIZMwKAChsHMcbEuG5v4xWReOAe4CKgEnhHRMpUdUtQtqXAYVWdIiKLgduAr4pICd4a6dOB8cALIjLVLWv7O+BuvLXRO6jqV4PO/QugLmj3dlWd1fPLHDjHWyBeABmXmUJmSgJbq+qjWS1jjBlwkbRA5gAVqrpDVVuAlUDnieUXASvc9hPAPPEW9F0ErFTVZlXdibee+RwAVX0NOBTupO74rwCP9eB6Bp2v0yC6iDAtL5OtVf27xKYxxgw1kQSQfGBP0PtKlxYyj6oG8FoNOREeG875wAFV/TgorUhENorIqyJyfqiDRORaESkXkfLBeEq1fRC9vQUCMC0vk23762ltszuxjDGxaygPoi/hxNZHFTBRVWcDNwK/F5HMzgep6v2qWqqqpbm5uQNeSV+Tn6T4OJITjv8qS/IyafS32uqExpiYFkkA2QtMCHpf4NJC5hGRBCALqI3w2JO4Mr4IPN6e5rrBat32emA7MDWC+g8obyLFBLweN0/JeC+u2TiIMSaWRRJA3gGKRaRIRJLwBsXLOuUpA65221cAL6n3JF0ZsNjdpVUEFAPrIjjn54EPVbVjVXgRyXUD+ojIZFfWjgjKGlC+Rv8J3VfgPUwYHydsqaoLc5Qxxgx/3d6FpaoBEbkeWAPEAw+q6mYRWQ6Uq2oZ8ADwiIhU4A2ML3bHbhaRVcAWIABc5+7AQkQeAy4ERotIJfBDVX3AnXYxJw+eXwAsFxE/0AZ8S1XDDsIPlvaJFIOlJMZzWu4Ia4EYY2JaRLPxqupqYHWntFuCtpuAL4c59lbg1hDpS7o43zUh0v4A/CGS+g6m9okUO5uWl8m6nVGPb8YYM2CG8iD6sFDfdHIXFsD08ZlU1TVxsMGmNDHGxCYLIH3kc4PonZ1ZkA3Ae5VHBrtKxhgzKCyA9JGv0d8xkWKwGflZxAls+sQCiDEmNlkA6YPmQCvNgbaOiRSDjUhOYOrYDDZV2p1YxpjYZAGkD46vBXJyCwRg1oRs3t1zxNYGMcbEJAsgfRBqGpNgZ07Ipq7Rz67aY4NZLWOMGRQWQPqgfSLFzs+BtJs1wRtI37Tn8KDVyRhjBosFkD7oaIGE6cIqHpNOamK8DaQbY2KSBZA+6BgDCdOFlRAfx5kTsijfbS0QY0zssQDSBx2LSYV4DqTduUU5bKnydeQ1xphYYQGkD7obRAc4t2gUqlC+y6Y1McbEFgsgfeBr8hMfJ6QlxYfNM3viSBLjhbU2L5YxJsZYAOkDX6M3E2/wWiCdpSbFc2ZBNmt3WAAxxsQWCyB9EG4ixc7mFI3ig711HG0ODEKtjDFmcFgA6YNwEyl2du7kHAJtyoZP7G4sY0zssADSB75GPxnJ3bdASid54yBvVtQOQq2MMWZwRBRARGSBiGwTkQoRuTnE/mQRedztXysihUH7lrn0bSIyPyj9QRGpFpEPOpX1IxHZKyKb3Gthd2VFi6/JH1ELZERyAmdPGsmrH9UMQq2MMWZwdBtA3Drk9wCXACXAEhEp6ZRtKXBYVacAvwRuc8eW4C1POx1YANzbvq458DuXFsovVXWWe62OoKyoqG8KRDQGAnDB1Fy2Vvmo9jUNcK2MMWZwRNICmQNUqOoOVW0BVgKLOuVZBKxw208A88S7NWkRsFJVm1V1J1DhykNVX8NbPz1SYcuKlnDL2Ybymam5ALz28cGBrJIxxgyaSAJIPrAn6H2lSwuZR1UDQB2QE+GxoVwvIu+5bq6RPagHInKtiJSLSHlNzcB1GQVa2zja0hp2IsXOpo3LZHR6Mq9ZN5YxJkYMxUH0XwGnAbOAKuAXPTlYVe9X1VJVLc3NzR2I+gHdz4PVWVyccMHU0bz+cQ2tbbY+iDFm+IskgOwFJgS9L3BpIfOISAKQBdRGeOwJVPWAqraqahvwG453U/W4rIHU3WJSoXxmai6Hj/ltnXRjTEyIJIC8AxSLSJGIJOENZJd1ylMGXO22rwBeUm8ZvjJgsbtLqwgoBtZ1dTIRyQt6+3dA+11aPS5rIHVMpBhhFxbAhVPHkBAnPL/lwEBVyxhjBk23AcSNaVwPrAG2AqtUdbOILBeRy1y2B4AcEakAbgRudsduBlYBW4DngOtUtRVARB4D3gJOF5FKEVnqyvq5iLwvIu8BnwW+111Z0dA+kWJGhF1YAFlpicydnMOaD/bbMrfGmGEvoj+f3a20qzul3RK03QR8OcyxtwK3hkhfEib/17uoR8iyoiGSqdxDmT99LP/+zGYqqhsoHpsxEFUzxphBMRQH0YcFXw8H0dtdVDIOgDWb9/d7nYwxZjBZAOml7pazDWdcVgqzJmSzZrONgxhjhjcLIL3kawogAhnJPevCApg/fRzv761jz6FjA1AzY4wZHBZAesnX6Cc9KYG4uPBrgYTzhTO8G83K3t3X39UyxphBYwGkl+qbAj3uvmo3YVQa5xSO5KmNe+1uLGPMsGUBpJd8Tf6IpzEJ5fLZ+VRUN7B5n68fa2WMMYPHAkgv9WQixVAunZlHYrzw9MaoPUxvjDF9YgGkl3xNgR49hd5ZdloSF54+hrJ399ncWMaYYckCSC9Fuh56V744O5/q+mabodcYMyxZAOmlvnZhAXy+ZCyj05N5dO3ufqqVMcYMHgsgvdDWptQ3960LCyAxPo6vlBbw0ofV7DvS2E+1M8aYwWEBpBcaWgKo9mwixXCWzJmIAo+/s6fbvMYYM5RYAOmF42uB9K0FAt4zIRcU57LynU8ItLb1uTxjjBksFkB6oWMerH5ogQBcee5EDviabZ0QY8ywYgGkF3o7kWI486aNZVJOGve/tsOeTDfGDBsWQHqhfSr3vjyJHiw+TvjGp4vYtOcI63cf7pcyjTFmoEUUQERkgYhsE5EKEbk5xP5kEXnc7V8rIoVB+5a59G0iMj8o/UERqRaRDzqVdbuIfCgi74nIUyKS7dILRaRRRDa51329vei+6u8uLIArzp7AyLRE7n9tR7+VaYwxA6nbACIi8cA9wCVACbBEREo6ZVsKHFbVKcAvgdvcsSV4a6hPBxYA97ryAH7n0jr7CzBDVc8APgKWBe3brqqz3OtbkV1i/6tv6t8uLIDUpHi+PncSf9l6gB01Df1WrjHGDJRIWiBzgApV3aGqLcBKYFGnPIuAFW77CWCeiIhLX6mqzaq6E6hw5aGqrwGHOp9MVZ9367ADvA0U9PCaBlx/d2G1+/p5hSTFx3HvK9v7tVxjjBkIkQSQfCD4IYVKlxYyj/vyrwNyIjy2K/8I/DnofZGIbBSRV0Xk/FAHiMi1IlIuIuU1NQMzRYiv0U9aUjyJ8f07hJSbkczX5k7iyQ2V7Dx4tF/LNsaY/jZkB9FF5N+AAPCoS6oCJqrqbOBG4Pciktn5OFW9X1VLVbU0Nzd3QOrW16ncu/Ktz5xGUkIcd7348YCUb4wx/SWSALIXmBD0vsClhcwjIglAFlAb4bEnEZFrgC8AV6q7r9V1g9W67fXAdmBqBPXvd/VNgX4dQA+Wm5HMVecV8symvVRU21iIMWboiiSAvAMUi0iRiCThDYqXdcpTBlzttq8AXnJf/GXAYneXVhFQDKzr6mQisgC4CbhMVY8Fpee2D8CLyGRXVlRuWfI19X0ixa780wWTSUmM55cvfDRg5zDGmL7qNoC4MY3rgTXAVmCVqm4WkeUicpnL9gCQIyIVeN1LN7tjNwOrgC3Ac8B1qtoKICKPAW8Bp4tIpYgsdWXdDWQAf+l0u+4FwHsisglvoP5bqnrSIPxg8DX2fSLFruSkJ/ONTxfxp/eq7LkQY8yQJbH85HNpaamWl5f3e7mfuf1lzizI5q4ls/u97HZHmwN89j9fYXx2Kk9953/h3dRmjDEDT0TWq2ppd/mG7CD6UFbfFOiXiRS7MiI5gX+Zfzqb9hzhj+9VDei5jDGmNyyA9JCqeotJDdAgerArzipg+vhMbvvzhzS2tA74+YwxpicsgPRQo7+VQJsO6CB6u7g44ZYvlLD3SCN3vWS39RpjhhYLID3kaxyYp9DDOXdyDl8+u4DfvLaDD/f7BuWcxhgTCQsgPdQxD9YgdGG1+9eF08hMTWTZk+/T1ha7Nz0YY4YXCyA95BuAiRS7M3JEEv/n0mls/OQIj67dPWjnNcaYrlgA6aH2LqyBfA4klL+bnc/5xaP56eoP2WXzZBljhgALID3U3gLJGMQuLAAR4fYrziQpIY7vrdpk66cbY6LOAkgPtU/lPtDPgYQyLiuFn1w+g42fHLEp340xUWcBpIcGYjXCnvjbM8ezaNZ47nzxYzZ8YtOcGGOixwJID/ma/CQlxJGSGN995gGyfNEMxmencP2jGzh0tCVq9TDGnNosgPSQN5FidFof7bJSE/nVlWdz8GgL/7xyI612a68xJgosgPRQfZN/0O/ACmVGfhbLL5vO6x8f5E5bfMoYEwUWQHrI1xQgYxCfAenKV8+ZwJfPLuCuFz/mz+/bhIvGmMFlAaSHvIkUo98CAe/W3h9fPoOzJmbzvVWbeHfPkWhXyRhzCrEA0kMDvRphT6UkxnP/VaXkZiSzdEU5e480RrtKxphTREQBREQWiMg2EakQkZtD7E8Wkcfd/rUiUhi0b5lL3yYi84PSHxSRahH5oFNZo0TkLyLysfs50qWLiNzlynpPRM7q7UX3xUCvRtgbo9OTefDqc2gOtPKPD71D3TF/tKtkjDkFdBtA3Drk9wCXACXAEhEp6ZRtKXBYVacAvwRuc8eW4K2hPh1YANzbvq458DuX1tnNwIuqWgy86N7jzl/sXtcCv4rsEvuXN4g+dFog7YrHZnDf185m58GjXPO7dRxtDkS7SsaYGBdJC2QOUKGqO1S1BVgJLOqUZxGwwm0/AcwTbw3WRcBKVW1W1Z1AhSsPVX0NCLWmeXBZK4DLg9IfVs/bQLaI5EVykf2lyd9Kc6BtSHVhBfubKaO5a8ls3qus45sPl9Pkt0WojDEDJ5IAkg/sCXpf6dJC5lHVAFAH5ER4bGdjVbX9lqL9wNge1AMRuVZEykWkvKampptT9Ux9U3QmUuyJBTPGcfsVZ/DX7bVc//uNtARszixjzMAY0oPoqqpAj56SU9X7VbVUVUtzc3P7tT7Rmkixp754VgE/XjSdF7Ye4DuPrreWiDFmQEQSQPYCE4LeF7i0kHlEJAHIAmojPLazA+1dU+5ndQ/qMaDqoziRYk99/bxCfnL5DF7YWs03Hy63NdWNMf0ukgDyDlAsIkUikoQ3KF7WKU8ZcLXbvgJ4ybUeyoDF7i6tIrwB8HXdnC+4rKuBZ4LSr3J3Y80F6oK6ugZFtCdS7KmvzZ3E7VecwZsVB7n6oXU02MC6MaYfdRtA3JjG9cAaYCuwSlU3i8hyEbnMZXsAyBGRCuBG3J1TqroZWAVsAZ4DrlPVVgAReQx4CzhdRCpFZKkr62fARSLyMfB59x5gNbADbyD+N8B3+nTlvRCN1Qj76sulE7hj8WzW7z7MV+57iwO+pmhXyRgTI8RrKMSm0tJSLS8v77fyfr/2E/71qfd5a9nnyMtK7bdyB8OrH9Xwnf9ZT1ZqIg/9wxxOH5cR7SoZY4YoEVmvqqXd5RvSg+hDTX3T8OrCCvaZqbms+tZ5tKpyxa/+yhsfH4x2lYwxw5wFkB7wNfmJjxPSkqK3FkhfTB+fxVPf+RvGZ6dy9UPreOCNncRyC9QYM7AsgPRA+zQm3jOSw9P47FSe+PZ5zPvUGH787Ba++/gmu0PLGNMrFkB6wNfkH/LPgEQiIyWR+752Nv9y8VTK3t3HF3/1V3bXHo12tYwxw4wFkB6obwoMi2dAIhEXJ1z/uWIevOYc9h4+xqV3vcHTGwf1sRpjzDBnAaQHvLVAhn8LJNhnTx/D6n8+n0+Ny+C7j2/ixsc3ddwsYIwxXbEA0gO+IToTb18VjExj5bVz+e7ni3l6014uvesNyneFmufSGGOOswDSA77G2OnC6iwhPo7vfn4qq/7pPFrblC//+i3+44+bOdZiT68bY0KzANID9TEyiN6V0sJRrPneBXx97iQeenMX8+94jb9W2DMjxpiTWQCJUKC1jaMtrTHZhdVZenICyxfN4PFr5xIvwt//di03PfEutQ3N0a6aMWYIsQASoeE0E29/OXdyDn/+5wv4pwsm8+SGvXz2P19hxV93EWi1NUaMMRZAIuYbxtOY9EVqUjzLFk7jue+ez8yCLH5Ytpkv/PcbrNtpg+zGnOosgESovQWSMYRXIxxIU8Zk8D9Lz+XeK8/C1+jnK79+i2sfLqeiuiHaVTPGRIkFkAh1rAUyjKZy728iwsKZebzw/c9w40VTebPiIPPveI1lT75v08QbcwqyABKhU7ULK5S0pARumFfMqzd9lq/PncQT6/fwmdtf5rbnPuTQ0ZZoV88YM0gsgETI13jqDaJ3Z3R6Mj+6bDov3nghF5eM475Xt/Pp217i/67eSk293bFlTKyLKICIyAIR2SYiFSJyc4j9ySLyuNu/VkQKg/Ytc+nbRGR+d2WKyOsissm99onI0y79QhGpC9p3S18uvKfaWyCx/hxIb0zMSeOuJbP5y/cu4OKSsfzm9R2c//OXWP7HLda1ZUwM6/bPaRGJB+4BLgIqgXdEpExVtwRlWwocVtUpIrIYuA34qoiU4K2hPh0YD7wgIlPdMSHLVNXzg879B46viQ7wuqp+obcX2xe+pgAikJFsLZBwpozJ4I7Fs7lhXjH3vLydFW/t4n/e3s1ls8az9NNFTMvLjHYVjTH9KJIWyBygQlV3qGoLsBJY1CnPImCF234CmCfeohmLgJWq2qyqO/HWM58TSZkikgl8Dni6d5fWv3yNftKTE4iLG75rgQyWybnp/OIrZ/Ly9y9kyZwJ/Om9Ki6583Wu/O3bvPxhNW1ttoiVMbEgkgCSD+wJel/p0kLmUdUAUAfkdHFsJGVeDryoqr6gtPNE5F0R+bOITA9VWRG5VkTKRaS8pqYmgsuLTKxOpDiQJuak8R+LZvDWss/xgwWfoqK6gX/43Ttc9MtXWfHXXR3dgsaY4WkoD6IvAR4Ler8BmKSqZwL/TZiWiarer6qlqlqam5vbb5XxNQZO2WdA+io7LYlvX3gar9/0Oe746ixGJCfww7LNzLn1BW564l027TliS+saMwxF8o24F5gQ9L7ApYXKUykiCUAWUNvNsWHLFJHReN1cf9eeFtwSUdXVInKviIxW1UGZ6a++yX9KPwPSH5IS4rh8dj6Xz87nvcoj/H7tJ5S9u49V5ZWU5GXy9+dO5LJZ462lZ8wwEUkL5B2gWESKRCQJb1C8rFOeMuBqt30F8JJ6f1KWAYvdXVpFQDGwLoIyrwCeVdWOW3hEZJwbV0FE5ri61/bscnvP1xSwL7Z+dEZBNj/70hms/dd5/PjyGSjwf57+gHN+8gLX/34DL314AL/NuWXMkNZtC0RVAyJyPbAGiAceVNXNIrIcKFfVMuAB4BERqQAO4QUEXL5VwBYgAFynqq0AocoMOu1i4GedqnIF8G0RCQCNwGIdxH4PX6OfaXkZg3W6U0ZGSiJfnzuJr507kXcr63hyQyV/fHcfz75Xxej0JC47M58vnpXP9PGZuL8fjDFDhMRy33NpaamWl5f3S1kzf7SGL51VwI8uCzl2b/pRS6CNV7ZV89TGvby4tZqW1mFdcncAABU3SURBVDZOyx3BpTPzWHhGHqePzbBgYswAEpH1qlraXT4bFY5AW5vS0Bwg0wbRB0VSQhwXTx/HxdPHUXfMz7Pv7+PZd6u4++UK7nqpgsm5I1g4I4+FM/OYlmfBxJhosW/ECDS0BFA9tSdSjJastESuPHcSV547iZr6Zp7fsp/V71dx7ysV3P1yBYU5acyfMY55nxrLWROzSYgfyjcWGhNbLIBEoGMmXhtEj6rcjOSOYFLb0MzzWw6w+v0qHnh9J79+dQfZaYlcODWXedPGcsHUXLIs4BszoCyARMAmUhx6ctKTWTJnIkvmTMTX5Of1jw7y4ocHeGVbDU9v2kdCnHBO4SjmTRvD+cW5TB2bbl1dxvQz+0aMQL1NpDikZaYkcukZeVx6Rh6tbcqmPYd5cWs1L26t5id/2gpsZUxGMp+eMppPF4/m01NGMyYzJdrVNmbYswASAV/7eugWQIa8+Djh7EmjOHvSKG5a8Cn2HWnkjY8P8nrFQV75qIYnN3rPq54+NsMLJsWjOadwFOk2SaYxPWb/ayJwfDVC+3UNN+OzU/nKORP4yjkTaGtTtlT5eP3jg7xRUcMjb+/mgTd2EicwIz+LOYWjmFPkvbLTkqJddWOGPPtGjICtRhgb4uKEGflZzMjP4tsXnkaTv5XyXYdZt7OWt3ce4uG3d/PbN3YC8KlxGR3BZE7hKOvyMiYECyARqHddWOn2HEhMSUmM7+jGAmjyt/JeZR1rd9SybtchnlhfycNv7QYgPzuV2ROzmT1xJLMnZjN9fCbJCfHRrL4xUWffiBHwNfpJS4on0Z4xiGkpifEdrQ4Af2sbm/f5eGfnITbtOcKG3Yd59r0qAJLi4ygZn8lZLqDMnphNfnaq3ellTikWQCJga4GcmhLj45g1IZtZE7I70vbXNbFpz2E2fnKEjZ8c4ffrdvPgm163V86IJKbnZzFjfCYzXVdZwUgLKiZ2WQCJgK8xYAPoBoBxWSksyMpjwYw8wGulbNtfz8ZPDvP+3jo+2Ovj/td2EHCrLmalJjIjP5MZ47M6xl8mjUqzlS1NTLBvxQjUN/vtGRATUmJ8XEdgaNfkb+WjA/UdAWXzvjoeenMXLW56+hFJ8Uwdl8GnxmXwqXGZnO627c4vM9xYAImArzHA6HT7z20ik5IYzxkF2ZxRcLzrqyXQxsfV9Wx2AeXD/fX8+YP9PLbu+MrOYzOTOX1cpgssGZw+LoMpY9JtsN4MWRZAIuBr8jM5d0S0q2GGsaSEOKaPz2L6+CzaF+NUVarrm/lwfz3b9vv4sKqeD/fX87vttR2tlfg4YdKoNCbnpjNlTDqn5Y7wfo5Jt3E5E3UWQCLga7RBdNP/RISxmSmMzUzhM1NzO9IDrW3sqj3K1qp6tu2vZ3tNAxXVDbz6UTX+1uPr94zJSOa0EwKL12IZm5lsA/dmUEQUQERkAXAn3uqBv1XVn3Xanww8DJyNt8zsV1V1l9u3DFgKtAI3qOqarsoUkd8BnwHqXPHXqOomt5ztncBC4JhL39C7y46cqnrL2doguhkkCfFxLhhk8LdnHk8PtLbxyaFjbK85SkV1Q0dgeXrjXuqbAx350pLimZQzgsKctI6fhaNHUJgzgjEZyTaAb/pNt9+KIhIP3ANcBFQC74hImapuCcq2FDisqlNEZDFwG/BVESnBW552OjAeeEFEprpjuirzf6vqE52qcgnemurFwLnAr9zPAdXob6W1TW0Q3URdQnwck3PTmZybzkUlYzvSVZWa+mYqqhuoqGlg58Gj7K49xrYD9byw9cAJrZaUxDgmjRrBpKCgUpiTxqTRIxiXmUK8BRfTA5H8WT0HqFDVHQAishJYhLfOebtFwI/c9hPA3a7FsAhYqarNwE63Zvocl6+7MjtbBDzs1kF/W0SyRSRPVasiuIZe65jK3QKIGaJEhDGZKYzJTOF/TRl9wr5AaxtVdU3sqj3Krtpj7D7o/dxx8CivbKvpGGsBSIwXxmenUjAylYLsNCaMSqVgZBoFI1OZMCqN3HRrvZgTRRJA8oE9Qe8rOfkv/448qhoQkTogx6W/3enYfLfdVZm3isgtwIvAzS4AhapHPnBCABGRa4FrASZOnBjB5XWtYx4s68Iyw1BCfBwTRqUxYVQa5xefuK+1Tdnva2L3waPsrD1K5eFGKg83sufQMV78sJqDDc0n5E+KjyN/pAswQYGlYGQq47NSyc1IthbMKWYofisuA/YDScD9wA+A5ZEerKr3u+MoLS3VbrJ3y1YjNLEqPk7Iz04lPzv1pJYLQGNLK3uPHGOPCyyVh49Recj7+fy+/dQebTkhf0Kcd1NAXlYK47JSGJ+dSl5WCnlZ7md2CqNHWCsmlkQSQPbSft+hp8ClhcpTKSIJQBbeYHpXx4ZMD+qSahaRh4B/6UE9+l37RIoZNpGiOcWkJsV3DOaHcrQ5wN4jXoulqq6JqrpGqo40sa+ukQ/21vH8lgO0BNpOOCYx3gsy47NSycs+HlzGZXl3o43JSCY3I9nmnRsmIvlWfAcoFpEivC/sxcDfd8pTBlwNvAVcAbykqioiZcDvReS/8AbRi4F1gIQrs31cw42hXA58EHSO6914yblA3UCPf0BwF5a1QIwJNiI5galjM5g6NnSAUVUOHW1xwcULMPuOuEBT18SGTw6zv67qhEF+ABFvXrHcjBTGZiYzJiO5I7iMcT/HZqZYoBkCug0gbkzjemAN3i23D6rqZhFZDpSrahnwAPCIGyQ/hBcQcPlW4Q2OB4DrVLUVIFSZ7pSPikguXpDZBHzLpa/Gu4W3Au823n/o89VHwLqwjOkdESEnPZmc9OQTpnoJ1tamHDzazIG6Zqrrmzjga+aAr4nq+maq3c8t+3wcbGimLUSHdM6IpKCgksyYjBRy0pMYnZ7M6PRkcjOSyBmRTHZaoj0bMwDEu6kpNpWWlmp5eXmfyrjn5QpuX7OND3+8gJREm1LCmGhobVNqG5o54DseaDp+ukBzwNcUNtAkxAk56V4wGZ2RzOiOIHM82OSkJ5GbnsyoEUkknOItGxFZr6ql3eWzjv1u+Jr8JCXEWfAwJori447fquwNsYbW2qYcOdbCwYYWDjY0u1cLtZ22t1c3UNPQfNIYTbuRaYkdQSVnRDIjRyQyKi2JkSOSGDUiiZFp7ueIJEalJZGadGp+P1gA6YavMWDdV8YME/Fxx7vNTif02Ew7VaWhOXBCgKkJDjb1LdQebWbbgXoOHW3hyLGWkK0b8B7QDA4woYLMyBGJ3r60JLLTkkhKGP6tHAsg3fA1+e0ZEGNikIiQkZJIRkoiRaO7nyy1tU3xNfo5dKyFw0dbOHS0hcPHWjh01O9+eum1R1v45NAxDh1t6biLM5QRSfFkpSaSlZZEdmoi2WmJ7n0i2alJZLm0bJfmvU9iRFL8kBnPsW/GbthEisYY8Fo3I12Lgtzu84O34NjhYy0cPuoPCjjeq67Rz5Fjfuoa/dQ1tlBR3cCRRj91x/wnzBDQWUKcBAWa44ElKygIZaclUjQ6/YTVNAeCBZBu1DcF7BkQY0yvJMbHMSYjhTEZKREfo6o0+ds40ng8yBw55sfX6OdIY4v3vtEFnmN+ahqaqahp4Mgx/wktnr89czz/vWT2QFxWB/tm7IavyU/+yNRoV8MYc4oQEVKT4klNSiUvq2ffPYHWNuqbAhxp9JMwCE/8WwDphg2iG2OGi4T4uOPdbINg+N8GMMBsEN0YY0KzANKFJn8rLYE2a4EYY0wIFkC60D4glWmD6MYYcxILIF2wiRSNMSY8CyBdsIkUjTEmPAsgXejowrJBdGOMOYkFkC60d2FlWAvEGGNOYgGkC77G9kF0CyDGGNOZBZAuHB9Ety4sY4zpLKIAIiILRGSbiFSIyM0h9ieLyONu/1oRKQzat8ylbxOR+d2VKSKPuvQPRORBEUl06ReKSJ2IbHKvW/py4ZHwuekAUm0tEGOMOUm3AURE4oF7gEuAEmCJiJR0yrYUOKyqU4BfAre5Y0vwlredDiwA7hWR+G7KfBT4FDATSAW+EXSe11V1lnst780F90T7RIpDZepkY4wZSiJpgcwBKlR1h6q2ACuBRZ3yLAJWuO0ngHnifesuAlaqarOq7sRbz3xOV2Wq6mp1gHVAQd8usfe8aUxs/MMYY0KJJIDkA3uC3le6tJB5VDUA1AE5XRzbbZmu6+rrwHNByeeJyLsi8mcRmR6qsiJyrYiUi0h5TU1NBJcXnq0FYowx4Q3lQfR7gddU9XX3fgMwSVXPBP4beDrUQap6v6qWqmppbm6Eq76E4WsK2AC6McaEEUkA2QtMCHpf4NJC5hGRBLxV72u7OLbLMkXkh3hrft3YnqaqPlVtcNurgUQRGR1B/XutvslPRrK1QIwxJpRIAsg7QLGIFIlIEt6geFmnPGXA1W77CuAlN4ZRBix2d2kVAcV44xphyxSRbwDzgSWq2rGuo4iMc+MqiMgcV/fa3lx0pHyN1gIxxphwuv12VNWAiFwPrAHigQdVdbOILAfKVbUMeAB4REQqgEN4AQGXbxWwBQgA16lqK0CoMt0p7wN2A2+5ePGku+PqCuDbIhIAGoHFLkgNGF+TjYEYY0w4Ef157bqMVndKuyVouwn4cphjbwVujaRMlx6yTqp6N3B3JPXtD/7WNo61tNpdWMYYE8ZQHkSPqgZbC8QYY7pkASQMm0jRGGO6ZgEkjI6JFK0LyxhjQrIAEkbHRIrWhWWMMSFZAAmj3pazNcaYLlkACaO9CyvDWiDGGBOSBZAwfNYCMcaYLlkACcPX6EcE0pOsBWKMMaFYAAnD1xQgIzmBuDhbC8QYY0KxABKGr8lvz4AYY0wXLICE4U2kaAHEGGPCsQAShjeRoo1/GGNMOBZAwqhvshaIMcZ0xQJIGL5Gvz0DYowxXbAAEoatBWKMMV2zABJCW5vS0GxdWMYY05WIAoiILBCRbSJSISI3h9ifLCKPu/1rRaQwaN8yl75NROZ3V6Zb5natS3/cLXnb5Tn6W0NLAFWbSNEYY7rSbQARkXjgHuASoARYIiIlnbItBQ6r6hTgl8Bt7tgSvOVtpwMLgHtFJL6bMm8DfunKOuzKDnuOgeBrtGlMjDGmO5G0QOYAFaq6Q1VbgJXAok55FgEr3PYTwDzxFjRfBKxU1WZV3QlUuPJClumO+ZwrA1fm5d2co991rAViLRBjjAkrkgCSD+wJel/p0kLmUdUAUAfkdHFsuPQc4Igro/O5wp3jBCJyrYiUi0h5TU1NBJd3spTEOC6dmUfByLReHW+MMaeCmBtEV9X7VbVUVUtzc3N7Vcbk3HTuufIsZuRn9XPtjDEmdkQSQPYCE4LeF7i0kHlEJAHIAmq7ODZcei2Q7crofK5w5zDGGBMFkQSQd4Bid3dUEt6geFmnPGXA1W77CuAlVVWXvtjdQVUEFAPrwpXpjnnZlYEr85luzmGMMSYKuh0lVtWAiFwPrAHigQdVdbOILAfKVbUMeAB4REQqgEN4AQGXbxWwBQgA16lqK0CoMt0pfwCsFJGfABtd2YQ7hzHGmOiQWP4jvrS0VMvLy6NdDWOMGVZEZL2qlnaXL+YG0Y0xxgwOCyDGGGN6xQKIMcaYXrEAYowxpldiehBdRGqA3X0oYjRwsJ+qMxycatcLds2nCrvmnpmkqt0+iR3TAaSvRKQ8kjsRYsWpdr1g13yqsGseGNaFZYwxplcsgBhjjOkVCyBduz/aFRhkp9r1gl3zqcKueQDYGIgxxphesRaIMcaYXrEAYowxplcsgIQgIgtEZJuIVIjIzdGuT0+IyAQReVlEtojIZhH5Z5c+SkT+IiIfu58jXbqIyF3uWt8TkbOCyrra5f9YRK4OSj9bRN53x9w1UEsL95SIxIvIRhF51r0vEpG1rp6Pu6UDcMsLPO7S14pIYVAZy1z6NhGZH5Q+5P5NiEi2iDwhIh+KyFYROS/WP2cR+Z77d/2BiDwmIimx9jmLyIMiUi0iHwSlDfjnGu4cXVJVewW98KaX3w5MBpKAd4GSaNerB/XPA85y2xnAR0AJ8HPgZpd+M3Cb214I/BkQYC6w1qWPAna4nyPd9ki3b53LK+7YS6J93a5eNwK/B55171cBi932fcC33fZ3gPvc9mLgcbdd4j7vZKDI/TuIH6r/JoAVwDfcdhKQHcufM96y1juB1KDP95pY+5yBC4CzgA+C0gb8cw13ji7rGu3/BEPtBZwHrAl6vwxYFu169eF6ngEuArYBeS4tD9jmtn8NLAnKv83tXwL8Oij91y4tD/gwKP2EfFG8zgLgReBzwLPuP8dBIKHz54q3Ds15bjvB5ZPOn3V7vqH4bwJvRc6duBthOn9+sfg54wWQPe5LMcF9zvNj8XMGCjkxgAz45xruHF29rAvrZO3/SNtVurRhxzXZZwNrgbGqWuV27QfGuu1w19tVemWI9Gi7A7gJaHPvc4Ajqhpw74Pr2XFtbn+dy9/T30U0FQE1wEOu2+63IjKCGP6cVXUv8J/AJ0AV3ue2ntj+nNsNxuca7hxhWQCJUSKSDvwB+K6q+oL3qfcnRszcvy0iXwCqVXV9tOsyiBLwujl+paqzgaN43Q4dYvBzHgkswgue44ERwIKoVioKBuNzjfQcFkBOtheYEPS+wKUNGyKSiBc8HlXVJ13yARHJc/vzgGqXHu56u0ovCJEeTX8DXCYiu4CVeN1YdwLZItK+bHNwPTuuze3PAmrp+e8imiqBSlVd694/gRdQYvlz/jywU1VrVNUPPIn32cfy59xuMD7XcOcIywLIyd4Bit2dHUl4g29lUa5TxNwdFQ8AW1X1v4J2lQHtd2JcjTc20p5+lbubYy5Q55qxa4CLRWSk+8vvYrz+4SrAJyJz3bmuCiorKlR1maoWqGoh3uf1kqpeCbwMXOGydb7m9t/FFS6/uvTF7u6dIqAYb8BxyP2bUNX9wB4ROd0lzQO2EMOfM17X1VwRSXN1ar/mmP2cgwzG5xruHOFFc1BsqL7w7mz4CO+OjH+Ldn16WPdP4zU93wM2uddCvL7fF4GPgReAUS6/APe4a30fKA0q6x+BCvf6h6D0UuADd8zddBrIjfL1X8jxu7Am430xVAD/D0h26SnufYXbPzno+H9z17WNoLuOhuK/CWAWUO4+66fx7raJ6c8Z+A/gQ1evR/DupIqpzxl4DG+Mx4/X0lw6GJ9ruHN09bKpTIwxxvSKdWEZY4zpFQsgxhhjesUCiDHGmF6xAGKMMaZXLIAYY4zpFQsgxhhjesUCiDHGmF75/wfZTRHPkst0AAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Dqp06bixpP_K"
      },
      "source": [
        "# 訓練步驟"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Uy3p4-uypP_L"
      },
      "source": [
        "## Training 訓練"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RlnrxiFPpP_L"
      },
      "source": [
        "from fairseq.data import iterators\n",
        "from torch.cuda.amp import GradScaler, autocast\n",
        "\n",
        "def train_one_epoch(epoch_itr, model, task, criterion, optimizer, accum_steps=1):\n",
        "    itr = epoch_itr.next_epoch_itr(shuffle=True)\n",
        "    itr = iterators.GroupedIterator(itr, accum_steps) # 梯度累積: 每 accum_steps 個 sample 更新一次\n",
        "    \n",
        "    stats = {\"loss\": []}\n",
        "    scaler = GradScaler() # 混和精度訓練 automatic mixed precision (amp) \n",
        "    \n",
        "    model.train()\n",
        "    progress = tqdm.tqdm(itr, desc=f\"train epoch {epoch_itr.epoch}\", leave=False)\n",
        "    for samples in progress:\n",
        "        model.zero_grad()\n",
        "        accum_loss = 0\n",
        "        sample_size = 0\n",
        "        # 梯度累積: 每 accum_steps 個 sample 更新一次\n",
        "        for i, sample in enumerate(samples):\n",
        "            if i == 1:\n",
        "                # emptying the CUDA cache after the first step can reduce the chance of OOM\n",
        "                torch.cuda.empty_cache()\n",
        "\n",
        "            sample = utils.move_to_cuda(sample, device=device)\n",
        "            target = sample[\"target\"]\n",
        "            sample_size_i = sample[\"ntokens\"]\n",
        "            sample_size += sample_size_i\n",
        "            \n",
        "            # 混和精度訓練 \n",
        "            with autocast():\n",
        "                net_output = model.forward(**sample[\"net_input\"])\n",
        "                lprobs = F.log_softmax(net_output[0], -1)            \n",
        "                loss = criterion(lprobs.view(-1, lprobs.size(-1)), target.view(-1))\n",
        "                \n",
        "                # logging\n",
        "                accum_loss += loss.item()\n",
        "                # back-prop\n",
        "                scaler.scale(loss).backward()                \n",
        "        \n",
        "        scaler.unscale_(optimizer)\n",
        "        optimizer.multiply_grads(1 / (sample_size or 1.0)) # (sample_size or 1.0) handles the case of a zero gradient\n",
        "        gnorm = nn.utils.clip_grad_norm_(model.parameters(), config.clip_norm) # 梯度裁剪 防止梯度爆炸\n",
        "        \n",
        "        scaler.step(optimizer)\n",
        "        scaler.update()\n",
        "        \n",
        "        # logging\n",
        "        loss_print = accum_loss/sample_size\n",
        "        stats[\"loss\"].append(loss_print)\n",
        "        progress.set_postfix(loss=loss_print)\n",
        "        if config.use_wandb:\n",
        "            wandb.log({\n",
        "                \"train/loss\": loss_print,\n",
        "                \"train/grad_norm\": gnorm.item(),\n",
        "                \"train/lr\": optimizer.rate(),\n",
        "                \"train/sample_size\": sample_size,\n",
        "            })\n",
        "        \n",
        "    loss_print = np.mean(stats[\"loss\"])\n",
        "    logger.info(f\"training loss: {loss_print:.4f}\")\n",
        "    return stats"
      ],
      "execution_count": 36,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "I3QNnLRNpP_M"
      },
      "source": [
        "## Validation & Inference 檢驗和推論\n",
        "為防止訓練發生過度擬合，每過一段時間要做一次檢測，計算模型在未看過的資料上的表現。\n",
        "- 過程基本上和training一樣，另外加上 inference\n",
        "- 檢驗完畢可順便儲存模型參數\n",
        "\n",
        "單看 validation loss，我們很難知道模型真實的效能\n",
        "- 直接用當前模型去生成翻譯結果 (hypothesis)，再和正確答案 (reference) 計算 BLEU score\n",
        "- 也可用肉眼看翻譯結果的好壞\n",
        "- 我們用 fairseq 寫好的 sequence generator 來進行 beam search 生成翻譯結果"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aLsINtUPpP_M"
      },
      "source": [
        "# fairseq 的 beam search generator\n",
        "# 給定模型和輸入序列，用 beam search 生成翻譯結果\n",
        "sequence_generator = task.build_generator([model], config)\n",
        "\n",
        "def decode(toks, dictionary):\n",
        "    # 從 Tensor 轉成人看得懂的句子\n",
        "    s = dictionary.string(\n",
        "        toks.int().cpu(),\n",
        "        config.post_process,\n",
        "    )\n",
        "    return s if s else \"<unk>\"\n",
        "\n",
        "def inference_step(sample, model):\n",
        "    gen_out = sequence_generator.generate([model], sample)\n",
        "    srcs = []\n",
        "    hyps = []\n",
        "    refs = []\n",
        "    for i in range(len(gen_out)):\n",
        "        # 對於每個 sample, 收集輸入，輸出和參考答案，稍後計算 BLEU\n",
        "        srcs.append(decode(\n",
        "            utils.strip_pad(sample[\"net_input\"][\"src_tokens\"][i], task.source_dictionary.pad()), \n",
        "            task.source_dictionary,\n",
        "        ))\n",
        "        hyps.append(decode(\n",
        "            gen_out[i][0][\"tokens\"], # 0 代表取出 beam 內分數第一的輸出結果\n",
        "            task.target_dictionary,\n",
        "        ))\n",
        "        refs.append(decode(\n",
        "            utils.strip_pad(sample[\"target\"][i], task.target_dictionary.pad()), \n",
        "            task.target_dictionary,\n",
        "        ))\n",
        "    return srcs, hyps, refs"
      ],
      "execution_count": 37,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aEXq75qppP_N"
      },
      "source": [
        "import shutil\n",
        "import sacrebleu\n",
        "\n",
        "def validate(model, task, criterion, log_to_wandb=True):\n",
        "    logger.info('begin validation')\n",
        "    itr = load_data_iterator(task, \"valid\", 1, config.max_tokens, config.num_workers).next_epoch_itr(shuffle=False)\n",
        "    \n",
        "    stats = {\"loss\":[], \"bleu\": 0, \"srcs\":[], \"hyps\":[], \"refs\":[]}\n",
        "    srcs = []\n",
        "    hyps = []\n",
        "    refs = []\n",
        "    \n",
        "    model.eval()\n",
        "    progress = tqdm.tqdm(itr, desc=f\"validation\", leave=False)\n",
        "    with torch.no_grad():\n",
        "        for i, sample in enumerate(progress):\n",
        "            # validation loss\n",
        "            sample = utils.move_to_cuda(sample, device=device)\n",
        "            net_output = model.forward(**sample[\"net_input\"])\n",
        "\n",
        "            lprobs = F.log_softmax(net_output[0], -1)\n",
        "            target = sample[\"target\"]\n",
        "            sample_size = sample[\"ntokens\"]\n",
        "            loss = criterion(lprobs.view(-1, lprobs.size(-1)), target.view(-1)) / sample_size\n",
        "            progress.set_postfix(valid_loss=loss.item())\n",
        "            stats[\"loss\"].append(loss)\n",
        "            \n",
        "            # 進行推論\n",
        "            s, h, r = inference_step(sample, model)\n",
        "            srcs.extend(s)\n",
        "            hyps.extend(h)\n",
        "            refs.extend(r)\n",
        "            \n",
        "    tok = 'zh' if task.cfg.target_lang == 'zh' else '13a'\n",
        "    stats[\"loss\"] = torch.stack(stats[\"loss\"]).mean().item()\n",
        "    stats[\"bleu\"] = sacrebleu.corpus_bleu(hyps, [refs], tokenize=tok) # 計算BLEU score\n",
        "    stats[\"srcs\"] = srcs\n",
        "    stats[\"hyps\"] = hyps\n",
        "    stats[\"refs\"] = refs\n",
        "    \n",
        "    if config.use_wandb and log_to_wandb:\n",
        "        wandb.log({\n",
        "            \"valid/loss\": stats[\"loss\"],\n",
        "            \"valid/bleu\": stats[\"bleu\"].score,\n",
        "        }, commit=False)\n",
        "    \n",
        "    showid = np.random.randint(len(hyps))\n",
        "    logger.info(\"example source: \" + srcs[showid])\n",
        "    logger.info(\"example hypothesis: \" + hyps[showid])\n",
        "    logger.info(\"example reference: \" + refs[showid])\n",
        "    \n",
        "    # show bleu results\n",
        "    logger.info(f\"validation loss:\\t{stats['loss']:.4f}\")\n",
        "    logger.info(stats[\"bleu\"].format())\n",
        "    return stats"
      ],
      "execution_count": 38,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ifHUFMbGpP_N"
      },
      "source": [
        "# 儲存及載入模型參數"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bwCGWvrPpP_N"
      },
      "source": [
        "def validate_and_save(model, task, criterion, optimizer, epoch, save=True):   \n",
        "    stats = validate(model, task, criterion)\n",
        "    bleu = stats['bleu']\n",
        "    loss = stats['loss']\n",
        "    if save:\n",
        "        # save epoch checkpoints\n",
        "        savedir = Path(config.savedir).absolute()\n",
        "        savedir.mkdir(parents=True, exist_ok=True)\n",
        "        \n",
        "        check = {\n",
        "            \"model\": model.state_dict(),\n",
        "            \"stats\": {\"bleu\": bleu.score, \"loss\": loss},\n",
        "            \"optim\": {\"step\": optimizer._step}\n",
        "        }\n",
        "        torch.save(check, savedir/f\"checkpoint{epoch}.pt\")\n",
        "        shutil.copy(savedir/f\"checkpoint{epoch}.pt\", savedir/f\"checkpoint_last.pt\")\n",
        "        logger.info(f\"saved epoch checkpoint: {savedir}/checkpoint{epoch}.pt\")\n",
        "    \n",
        "        # save epoch samples\n",
        "        with open(savedir/f\"samples{epoch}.{config.source_lang}-{config.target_lang}.txt\", \"w\") as f:\n",
        "            for s, h in zip(stats[\"srcs\"], stats[\"hyps\"]):\n",
        "                f.write(f\"{s}\\t{h}\\n\")\n",
        "\n",
        "        # get best valid bleu    \n",
        "        if getattr(validate_and_save, \"best_bleu\", 0) < bleu.score:\n",
        "            validate_and_save.best_bleu = bleu.score\n",
        "            torch.save(check, savedir/f\"checkpoint_best.pt\")\n",
        "            \n",
        "        del_file = savedir / f\"checkpoint{epoch - config.keep_last_epochs}.pt\"\n",
        "        if del_file.exists():\n",
        "            del_file.unlink()\n",
        "    \n",
        "    return stats\n",
        "\n",
        "def try_load_checkpoint(model, optimizer=None, name=None):\n",
        "    name = name if name else \"checkpoint_last.pt\"\n",
        "    checkpath = Path(config.savedir)/name\n",
        "    if checkpath.exists():\n",
        "        check = torch.load(checkpath)\n",
        "        model.load_state_dict(check[\"model\"])\n",
        "        stats = check[\"stats\"]\n",
        "        step = \"unknown\"\n",
        "        if optimizer != None:\n",
        "            optimizer._step = step = check[\"optim\"][\"step\"]\n",
        "        logger.info(f\"loaded checkpoint {checkpath}: step={step} loss={stats['loss']} bleu={stats['bleu']}\")\n",
        "    else:\n",
        "        logger.info(f\"no checkpoints found at {checkpath}!\")"
      ],
      "execution_count": 39,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tYkO9LltpP_O"
      },
      "source": [
        "# 主程式\n",
        "## 訓練迴圈"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4p7oPThZpP_O"
      },
      "source": [
        "model = model.to(device=device)\n",
        "criterion = criterion.to(device=device)"
      ],
      "execution_count": 40,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HvNfLwAapP_O",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "57f203de-3d56-42fa-bb2e-2695d97ad56c"
      },
      "source": [
        "!nvidia-smi"
      ],
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Tue Aug  3 02:34:27 2021       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 470.42.01    Driver Version: 460.32.03    CUDA Version: 11.2     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla T4            Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   40C    P0    26W /  70W |   1116MiB / 15109MiB |      0%      Default |\n",
            "|                               |                      |                  N/A |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gytSmdH6pP_O",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "16a49b56-3fc5-4939-d9d2-fa48f45e22c1"
      },
      "source": [
        "logger.info(\"task: {}\".format(task.__class__.__name__))\n",
        "logger.info(\"encoder: {}\".format(model.encoder.__class__.__name__))\n",
        "logger.info(\"decoder: {}\".format(model.decoder.__class__.__name__))\n",
        "logger.info(\"criterion: {}\".format(criterion.__class__.__name__))\n",
        "logger.info(\"optimizer: {}\".format(optimizer.__class__.__name__))\n",
        "logger.info(\n",
        "    \"num. model params: {:,} (num. trained: {:,})\".format(\n",
        "        sum(p.numel() for p in model.parameters()),\n",
        "        sum(p.numel() for p in model.parameters() if p.requires_grad),\n",
        "    )\n",
        ")\n",
        "logger.info(f\"max tokens per batch = {config.max_tokens}, accumulate steps = {config.accum_steps}\")"
      ],
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-08-03 02:34:47 | INFO | hw5.seq2seq | task: TranslationTask\n",
            "2021-08-03 02:34:47 | INFO | hw5.seq2seq | encoder: TransformerEncoder\n",
            "2021-08-03 02:34:47 | INFO | hw5.seq2seq | decoder: TransformerDecoder\n",
            "2021-08-03 02:34:47 | INFO | hw5.seq2seq | criterion: LabelSmoothedCrossEntropyCriterion\n",
            "2021-08-03 02:34:47 | INFO | hw5.seq2seq | optimizer: NoamOpt\n",
            "2021-08-03 02:34:47 | INFO | hw5.seq2seq | num. model params: 11,469,824 (num. trained: 11,469,824)\n",
            "2021-08-03 02:34:47 | INFO | hw5.seq2seq | max tokens per batch = 8192, accumulate steps = 2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": true,
        "id": "VJtMvf9TpP_P",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "f857b6d1db15485096eac510203e20d4",
            "4a6829019582497a8a2a56359a1bc1b4",
            "1354ea065af247089ab86cbbb2cf05cc",
            "ab2069c9a95f4db3a0a170059c5f16a8",
            "8cf83a4c9488420e9e7c0eabd4650f2d",
            "d29f5bfb768e43fa85424722c25d0e7e",
            "d7537cc2b9564c43b89c04af54aaab56",
            "770c3d6fd0834979a80a9f178e15d0d4",
            "9ecdb4186dd14b37a2d87f1acaa827ad",
            "6bda332921c547f680ad29c19a3819a1",
            "27ba67eb61614c97a084a3b1526f5a4d",
            "6aa9f53a019d410487e8295cdb535f46",
            "cab77ce0c7d94dea8d0001e9a08f7b9e",
            "4adda82d74c34a95a92b61dc15f3509e",
            "38c62ebcb3574038ac9844d9cf12d3ae",
            "de1e10dea8c240cca8eedb8c2ce52f15",
            "97422a5d2c104b51b78289c533aa4140",
            "aecea25cc31f4712873fa958dd0b1cd8",
            "2bfe08d6a9104cf6a9d3059215b0f7f4",
            "38f76a378612411bb7fd54417739cff4",
            "5d5f2ed804234e80a606faec322d4b49",
            "9c17c8976c1b409f8a2fda6422f67151",
            "f06f0f3c4c974de890b19aa0cd07c1e6",
            "9509eeab126246c1bfb1252c2f2e6da1",
            "5efc4463901548d0ae9f3c6e390305fb",
            "ef61d88357d74800abe2211213875c8e",
            "e57e9bd98d7c491190b99a8e48e4f288",
            "b39cf53022804d70a2a92b65e298699d",
            "b3b62d4dc6f94b539a70bed6c213ab64",
            "94f00c3001b846949f138a599aa38eaa",
            "585132bd1ec74b19b39bc1d56c3fc4c6",
            "56442fd8c35044419ad6898c5c45c66b",
            "a1c8b59a443140ffaac9c55deff7fc17",
            "cf7a89a621104564a1420c60b8e3967a",
            "73376e7e5cf84ba8949e7644a4b7a9e3",
            "9461c19314994ab798f5078b91475155",
            "dc01a3f28702451bba06c2ba2a99bcd5",
            "590fbc6632344811954b770e84093734",
            "dc0f264da2a141cfadfd3184133ac52e",
            "c2c4877ef5b8464b9b8af0ed813cffd9",
            "a77fdf78a3c744a9953aa32ebc56ba6e",
            "8c4cca19a54344ce92baf365686edf34",
            "210e37cbc63941c78861350b27b5a767",
            "06ec60c0099a4741b0a9c25cdd26752e",
            "1527fc05da3b42f897ab722d6ec3ae94",
            "761a321f0b62499fb85ed6f2e3fd28a0",
            "ecc833c7be1241e3a3b8c42b0a0413f8",
            "6736f29d704b4fcda53aabff067e50f3",
            "1dd956561a4949629cc483805c4d30ca",
            "7540cb5f8c53476a9c95c4edbfbb4dab",
            "37d8d63f5ac14651a1d23bb2de7048e8",
            "cc4cbc495137415ab68470fbac5d0a21",
            "25274892efe0411ebe57d399f0f6d3b1",
            "73f74cab664c4e818a8940e6cb455139",
            "6cea2d3e1f584834b6ddda2047fc2081",
            "7ed74ada9b634f4c9a6a08359a54e309",
            "7a3fa24175464695a5d8bc5144fcc5d8",
            "664ad871c2d64964aa3a5f7c38e523e8",
            "d12d47d4d90d46d194ea2ca282a16ee0",
            "a965289cf7d9449cb982467cf6d35c87",
            "eb746a8bce534f4eb8b1eafce1c8e42c",
            "61db40e9e5b74cdbbbbde6292b5e1009",
            "8e7dd65ae690444c9ab5afbf938159a2",
            "06f51b1647d04b98ac1505b9d9541a84",
            "9347496d609b4d22a7b13ecf0bb44320",
            "30a2db8f759147a18cc1523123183acd",
            "877616819dfa4a2185230dcd5b4b393a",
            "44f37691eaba41c488d99c2fe805bad6",
            "5611d68808e9408f9e668d620377331a",
            "dccad50d188141178b2e18ccb8538a43",
            "869ab53cc26943599d01be2cbc80e937",
            "4186d8f12a22414f88e7167988955bdb",
            "ece3c14c188243baac9d1682c8005859",
            "3a27d8a9408243faae99dc7ad12e7d33",
            "19e99c5edaa4472487c43cd879cd3c72",
            "3b0bf4a83d2548a687adf3f10b2b122b",
            "b7dd9f38300d4fdaa11bbf499b4572a9",
            "13f6ac24e3524b0ba037b20467b9c5fe",
            "d645b28b4b2342efb3d739f2fd2b6709",
            "fff9be908d0742b1a58d85fc462d28b9",
            "e3898759a14d4e21aea94e421a4bcd29",
            "c2b648403f8e45b1be1869958e40f3ae",
            "cd61247b7d394b5da7210bae1acf4e8e",
            "d4f93415846e471691b6a4bdbd09bf11",
            "1b160eb55a5c4e43bad219130fd43dc9",
            "099344472f1d422f8f93e7d05c308e2e",
            "324ede7d621f45d397bdb41970360b43",
            "411006be765b4aa4ada028f055639080",
            "afd568333e1641faa1ad279c13d1c630",
            "3aeac31a2fdd449382145a7702a0107e",
            "b455e18a90974fb7a6f0169ca880b9e8",
            "c6513dd1e6bb46ba9661fc120228b3ce",
            "e36a560e62e147f0a5ba17815a6dd385",
            "d01c0a37e0dd4f16893d5e6a5f60a162",
            "e48264c1f23d4feaa7802f35c101da03",
            "fb03f85f832e4dceab9c414dc17ac47e",
            "864094c044844e0aa70aa6ee04d14a73",
            "9cbce7e0ddea46f6aa9a58ee226608a6",
            "f6b82034f5b444c6badb80045fd4d25f",
            "56b6a058ebdd47adb0d0de1b95694f56",
            "c071f45623f94bfa8404097847f73aa5",
            "4c4afdcd754f43ed95a604a4945f4828",
            "11c9dad426ff46c7beeefbcfecdabf11",
            "ed33815ca4d84ffd9820132258ecb3a5",
            "8d2daceaa80041629ad96f67f50ec2d7",
            "723ced0add5c4612a73479a5a8f2eaf1",
            "3d8c2b727da3450ab552aa17dbbd08fb",
            "09eeb1f462cc40bdabc46ed73710cf62",
            "ae19af82b04f437db4f489842936fb70",
            "6f966090fdad4233b9230d053ed5e49b",
            "135b5722644849c5829f1b51a69c9734",
            "248bb2ccbfe14f5984aa6eab1fd1c698",
            "1e808665cdfa4a5998f26c3d4b51c7a5",
            "3f4261ad57314809bc0c8b48f9975325",
            "729c0b469e774502abfd47d6dbf73167",
            "5fd99d62217342ec9815b4a1b9b40e5a",
            "a47ac6f3f1054ad7abb336f583105469",
            "61600384caf64d2297c4c38279a72bb1",
            "c95435ed0843475e8345e3e725a7e24c",
            "7e1dce0e22354f7f8fd285a4c3988427",
            "da42ef922b904b76811fa04efbea3ba9",
            "9cc9be5dbd1c457982023de386170d58",
            "6338507297a443dabd413a97799b45f7",
            "b95e56348f3042ef95b1643d5ff2d64d",
            "e0a7cd2190b64253868e09796c366a22",
            "4bca9d40b7c149f1a5ab3ce6c32bab67",
            "b23655f2d9aa46e7906445f6dd287a22",
            "52b8448b765b45b28eda7083a492dedd",
            "3490001381064d6b94e096891c8b656d",
            "17b7d18d68a94d36abd10fd4b1d5a420",
            "2299165bf4b0431ba4e2c1d71cc4d655",
            "73bc525624c14f978a2867a4da18c24a",
            "707e0846832b4b88b807b46edc3bf8be",
            "cfcb419a0705446ebde311bcc6024cd8",
            "094013d7f9d448e39bc0034c943a76e8",
            "e6e13878f94143669b6826511cb74db3",
            "ef66637c0f884002b233737b7e458feb",
            "13823c80b8c34f90b01dc8e53de1ed31",
            "b0fa20e522514c53a0a6d8140a948f15",
            "36e08f2f0b1f425fa950c2579f82d9ff",
            "6bbd28d29b244995a37da0e0636d24c6",
            "478062b7c07444f19686bdd9a3ca40a9",
            "c82263539410440cada74d2b50afc0ea",
            "9e42e6ad779641678b2e77f126fa162d",
            "d8ee47410b0a4556a0163485c26a1429",
            "f707df215a0142edbba2b8842b698599",
            "a7f4ebb409e5431981d3b7dac979931a",
            "21f7b29bf2814f209280d34b1a8d07a7",
            "105ddf9aaa3549bba1e314f3f8266a84",
            "de8f58d2112447e18fb98ed2a690028c",
            "9235581a300c4dc1aeca1c650669ef52",
            "d101a5702e6e4860a39b968fe75ee76d",
            "582a89642fa64656b63071f49b22796d",
            "a98a39f0ee6e467882809c23f0397d81",
            "e49d63f1446d4439ac439f9e3f86d684",
            "51d6496f7af349508b3a383c48bf65a0",
            "485c28bd5ff9423e9236de81cb7324df",
            "31a03080aa0242c9b69b6f95036fb291",
            "1d3f98937ea94d609ad800d44450eb84",
            "9c112d80f73e45aea6bee9bb67d60613",
            "fafc50e23a88426182fd07a779935b31",
            "b0830c9b0d9e4d9fa0f9558c15d5a11f",
            "b07e5bb52b3f4a1e8a8a43372fab881c",
            "205206e946004a96afe7833dac832034",
            "a2cd48d861474c0aa1872356949e046b",
            "bf6cacaeb81549aeb296d5262214587f",
            "41e1c6bef2cb4164ac87e04b05084e88",
            "3ba8a17a4cbe42388c23e44bef6ff64e",
            "b1794d354a864a7aabc11c5d2254fa13",
            "e63605bcc5f744e691bca858b4cedb3b",
            "5bfb1d2e0d484d4faea590975d73e282",
            "1885706b279d4d3c93f5892bcc1b850c",
            "be77f324e1c74fdd8e1ec101de1f4ab6",
            "522e647e4cbe40af9f3c7b0bf8024bf6",
            "600377cc97ed4963ac8974554c59ab91",
            "b82be97fb1cb41878c237d047e95b334",
            "3d3da426d14f4da098753388612a6f4c",
            "f7f9c2583da047239f090e3d0c359195",
            "a33210c7a85a45129b41ed523ba4d288",
            "c70e099e775e4d84b2ebb58fde27a574",
            "176267925e1b4f60a135e64250029b04",
            "4d7ca84f014c41deb60285969de910f5",
            "74f2649ea45749809be2c4498d92952d",
            "3bf0f082e18344c8a58ff0f74af1d3a5",
            "d23b4df531884201911c24998813416c",
            "5c4db674f0c0412fa37b41aba88df130",
            "27043264e83d43ef804105614e24c6db",
            "32d647faa2cb4034939b9e7085515046",
            "50827286682946c8860b9e45f9621f64",
            "799670136ce140a68835e6f548b255f9",
            "41b9cd1fdb63436fb4d1cf6c5f1434d4",
            "7639fb2acc764e918a73291206db55b0",
            "056fc96039164220854a7e0e9e8c2c0e",
            "a08cfe88e4ae45378a8a993292e73232",
            "ce9c5e2d3bbe4ed9a02671bbc322e626",
            "f331275dd6fc4719a9f75bee084c7bdd",
            "2ad3288b4bf040aca232d6a9530750dc",
            "34a6a01c2b914d209df92429b60f2f8f",
            "adf383f9d1274e409849405b1d73ce51",
            "ae02c0c0fd304e079528b9b255184790",
            "7ba282ed8b234aa5835fc0f46b0cc9b4",
            "aca9412a528743ff9a0459ce9f258e72",
            "cfe9820e6e464d6180d0876508d2cb7b",
            "ae8bfc940dab40fcb714b3ca239a9eb6",
            "db3d88e3a5da41feb05a0e1ec89aefe5",
            "99e3e37be7304cb0b03ee8d0f268d544",
            "50a35b5286874cc8ae0e919c2690f0d4",
            "43786826fff2466e9a2a18439a5a0544",
            "db1f4908985f428bb8d5413f876f6d93",
            "8f62a6bab64b4bc2bae81230dc7bdb6b",
            "541c657fc4a04a258b5a9300973a67b4",
            "1eb61e2aa5444b79b9e800fed7e0e252",
            "e507043b9ff344c180996a771b918af4",
            "108084cd0c0b488f82f25f6b6734315c",
            "699fd30fa0f14ac887818f4dbbc21af8",
            "3393eb56a31b404f8af7ca5db405d367",
            "f0ba7e145d0d4971be09662f2e2fbf3a",
            "4334aff5f56f45238b30de33aa9157ae",
            "98605c2b74f84fd797763af808cd408b",
            "c1a89341e52c49b8b6a2b50219ec5bb7",
            "b21049cf1a284d6a85cf5dfe0c3374f5",
            "b5f6875abb2e42b39034222df9d6fa3b",
            "6c4bf83ed932435e8f2760b931b098ee",
            "6716b40c827b462385c72805dc6c130a",
            "523d21c12e8144fe85d2bc5d8b8b87de",
            "e9368cfc54c848d9bd337845b2b19f61",
            "b25dc878560c40d281f9086b90fa2d0c",
            "d8ae2c801c9d4720b284f5b5d600a123",
            "745b288a158d4459a52bc44e60740079",
            "58d1aa331fa746308db890ea03f11dc7",
            "0be3265c7cfc47a48ada725efbc96417",
            "31d3edd7d9f94439adafd80473a9a85e",
            "cf92a0bfd2654884aaaf30dc7941aaf0",
            "c396333375de494e83fc7c2e70a97dd1",
            "79440766449a497c9c7d3684b9a70996",
            "249ce3d647604fdb842d2975a6addfaa",
            "2709d8ff97e847dcb788d497ea7688f8",
            "b6a3415e85124a2f88d2b0e250b40be1",
            "4fd6c7dc428e4d53a23a300b89918f3c",
            "05b7528ddc1d4c0d9b90690a9fc8a383",
            "aa8b0134139a4514bd938aa40df3314a",
            "48f34dba9634479b878d75a6859d9fa5",
            "08527c2450fb45b48fecfe2179166a1d",
            "31ee05b34eb04527ac0393a84db9ce64",
            "184cb124eeab463a99192375b7b1c3d1",
            "c44fb1c1e2cf44dcb1107dbdfd32fdd5",
            "091a0d1bd10e4492b3bf2ddb03edfb70",
            "50ddf8942dde4614b031e75b35d47924",
            "daeb3902e30044cdb74f8497d5287c31",
            "f8fce735712b4d83a07e199648b70362",
            "756820c04d4240a7a64eae01f927e574",
            "c18e780f7b414f01a347c3fbfc31e930",
            "93954e14055e4f6a97765bbe8417ac8a",
            "b8ea272386504dff947534417769bccc",
            "3fa3b12eee074d72abcb398e5a771e45",
            "236d5387f70f4f81982b1e1edf952a82",
            "8453f3413a55457f9ff114d37bceee39",
            "28f04de6c0d44e22852678ddd8ba474b",
            "f3bb385458f54dbcbb2db9c1a205c779",
            "167ef8c98d7742d09c60253e3ecb2dfe",
            "cdd8889aaacb487e809734482bacbe04",
            "7300647e149c4edf801b190e29fb52e0",
            "1e4b27ed3dc948169c272bfb201a6961",
            "654ff33ec18046489c3ccb48f4ef628b",
            "e6e392d13e9545bf987a61139d5445b6",
            "736bf193324f45bb89f154b7d2510ab0",
            "aecbb89d2268479fba0b126b1b93c19d",
            "ff6166e0d1bd4722b4007a9a3e7eaa31",
            "f63ab431f4a74a109b6bf1fdf12cb9f6",
            "8c31ce1324fd484dbd52981a5ee426b6",
            "00ec4615653d4abaa02e449063998014",
            "cdc3547313644b65aea412624585926f",
            "6022dba9c0fc44928150f03e4d0707f8",
            "c998d4f29cc34995860f46282c7b7138",
            "b1d6037240cd4696ac04c11070574544",
            "dc6f67cf20944a568126e44c65656164",
            "99a6e7782a7c468594583dc52285b8ab",
            "3a13dd610a584ea1bc9de59411e8459f",
            "1d8ec79053314cd48730f6ed43b76d96",
            "5f742d7673954b628f7d2ceb809b2ab4",
            "1046bb04ed3540ffad21248155f92a59",
            "c4cc9cc069924334b86d3b8d3f3f6cad",
            "179f7743beac48e4b72ed2446b02d77b",
            "02d41d1e5f254064bf2da346629f6ed4",
            "c64f8b6f3dec4bab9c970f9db13b4a1d",
            "c0780f1d5d3e469e88b7444f4e3e4278",
            "fbdb59a8f879435f804bd60b394959b2",
            "ac9f1d823fcc4e6696605c4c29e065da",
            "30bcbe1ee7f84a8ca8f9d40f77be522b",
            "dd14cb45f3594a0a9b7e0f14f9515e58",
            "8330a5ba7a3b4ae9a6a134d72f36b8da",
            "a67268ea3bfb41f0b0b5946399fc18ac",
            "2f2f7306afba4cc3a18ac26cb83d1e78",
            "61a5c0e56e8a4e7098f519b15a5f84e6",
            "3897f59d5536406e9c670c5be54a9867",
            "9925fe9372be4080b5122d5b47f38d78",
            "13eba79cc9f04134801a52f71881e8b1",
            "bd1c4ab320be4be6946ea73a74328837",
            "1651e4fce18046bbb2cfd7a86dde77ad",
            "5870ac198e4d4d7bb6e1d6f126a07266",
            "e475847bc2fa4954a1fae4b644da3b6d",
            "a8f6f7330fc04ac4a67bf2e68ba2ccb1",
            "c6f6a0f898ce411686cf966e3644fe06",
            "83133b3cd0e748208dfbecdff338af83",
            "c7bca190a59d44d0b41399002a2accae",
            "367a68bb6f034d8fbfc72228cc6e6f69",
            "b4c810084d4d4eb98c49118730578356",
            "ffd1ef53347447858f3e1078b5c1b1bb",
            "111c42d590054d519ccda957bd6abe19",
            "d4168759ae6541899aa472d63c5d7e61",
            "e20d80173a08410c8c67e96dc4c84e17",
            "aebc6fddfd9b4b5b8aca75d9b85cad10",
            "df8ad6dae6424015baef44d5e019d13d",
            "45a944d3204a4bb8a724d1e12046e2c2",
            "f3e1c80ac9c0468fa0f77e16b711153a",
            "f967aec337694a94b8b45bfd6358ed28",
            "bde4bb315d6041cea08fca0e255a2f11",
            "b656d24293ee4964900d1126056780e8",
            "88a3f3e62b2e41df87b8105688801b98",
            "e2e20571c95e44cb8319f2454d64c2f6",
            "7023d21c52f140a7a119edbcd783743b",
            "77be2ea8783e4bf9af2971561b25588c",
            "5328e71f61414a47a2fa30950d89f1ed",
            "0f246499e9604af58c520e5ffe199bf4",
            "735905ae59654c9c98d94b7e0504ed4b",
            "0f5143ecd56248d689c74a1aaf331687",
            "6e503e5c5e3f4b0daa0a7cf8736d439b",
            "6d0abe069d764f249d92bbc113642775",
            "c9f024091d8841d99ad01854fc1cb691",
            "cee15e5cc0c94d4188637c25b82e34b3",
            "48c22170fbad4a83a7f591e63ff15c68",
            "8d90770b6d794b2fb3c0bafc798c010d",
            "49584d6498814cf1b8320f973b598ed5",
            "7333086d247641eba70eced3d21fbc43",
            "fb286ed5e96e4a2981031f0bc03e7ff0",
            "ea2c83790ba14303a461fafeb6a66249",
            "6c8a3569bfa2444681b7a61cf648df9a",
            "5945abbd4fdf44f2a7f92131710d028d",
            "c9a01fcc232c46a78d5407e5731a7436",
            "4b8dcee59f974455876a63334071c639",
            "025d0ee242d54468af1440337dcc511c",
            "f8c4d86918704308b4b37f6db98b9dc0",
            "2bed57b412344be5b4d6bdf5253d1ccb",
            "1c758e4db7bf4bed92458d8d580fe470",
            "c47bad70e69b46e6a76550d86099af03",
            "04f40847ce10479ab8429eb6d15c251d",
            "58255918c2b9489aa7738d54992442e0",
            "fba12efe93d64dd1834b871237005eb3",
            "18c46b9edaf84f7db2669f47679138b9",
            "5037bebbabd14c61b1169683c7aafe75",
            "39a99785c8364475886de558886ca1fe",
            "ad3f834155174d028b21bc0f6175c232",
            "44f6a968267b446d93cc9abd73e1d927",
            "18977684aaba4d51bb098816e2f9556c",
            "8285cfee4d4442c4aeeec6913fdb45b3",
            "fd72d7e216cd4dcbb0b2d7c8131737aa",
            "7925edffcf0a4162aebfa12abdb5f935",
            "18bf824e805d4c7087e3b002994dae54",
            "4b10a63056a444f5805b5d4d52361e48",
            "129e1468e741449ab8edb3b85676884d",
            "9c2bcd73d15948f2a01be41f75da486b",
            "e0d7c278bc384bbbabf8bff598bb45fa",
            "dbd43ba14a7f4597a9fd848396be5bdd",
            "f2c36b539ceb40f28dd6785fab672667",
            "ce95a0f5ebc343e89ed11139de9c42af",
            "51d07c1531674bceb9e7a11edf4b610b",
            "a84d24ed971b4040ace8316ee2c95121",
            "3a2a2419b69345ccbc5b89f2630291fb",
            "fcc5d624f7e3492899da67b5728d5398",
            "00ff3cfd665a4b82a003d24441cb57d9",
            "f0b7118b96124c9ab39429390d37c9eb",
            "0fe5e5b67ea64e2ca29c4972d85f800b",
            "89b1788bcaae48389f0ec073e54ca6a6",
            "4209ea80cd5147e5b585ca90211277a0",
            "9e84965f4fb848958abd801776086b59",
            "5d74c464f4844f00ab2bf36ede064fd9",
            "c8645e6887fa43ef83ad37dd930e4c8a",
            "62e12316eb3d432184c8edfc9ad2e1a3",
            "0b4c2dfacceb4a08be373086c10db2ff",
            "4587798f21d64b8f87e0863c1d05d727",
            "258f21d530d84ff78567150fc3a9d617",
            "0506a06b87de44e99bb886f6d4ea41f4",
            "6110ad0c90db4ecd83f5a8705b12fe55",
            "3040776c80ef45c4bd35faa073f423c8",
            "71872ce80eba4f51b0c79916b3541cc4",
            "1ac63be73eee4d699a10f14e9605be46",
            "5b036be977bd4e76ac340219bb40b6e8",
            "8c100a2482f940af91c0123e92c76540",
            "2f747fa318e94c4090d856aa813aa385",
            "fdfaff780a2644cdb537c0895fcafd7c",
            "72e0c1891ce64c7b84e3a43e31985b85",
            "a5b5467ccb3541f39edcb0773a708044",
            "c7bfaca199274391866ec46a23bdedf8",
            "8b028c27e7ad446b982dc3c9b0f0cfaf",
            "64f8c8c4d380489cbc5d1fb3adc56674",
            "97d284308f6147aca16faa5495c0c745",
            "d6270c3ab9214303be0ca78062a2716e",
            "918e2ba46396421d8267107c37ca1bd1",
            "ae96332e3b564949930c49d38923aeea",
            "5b6a11ddb5464af38a76496a78e41b50",
            "9e0fa3454b27483da48b8452bda494b1",
            "5fbf7da508e348a286efdb23061c73df",
            "dab65c11274a4141a994cbdb1d353faf",
            "659a14c602ee471e8539004c7be4a1ee",
            "f02af61894434e8a89962bae16df267c",
            "bfab2726ad484a0fab703e9194230ce8",
            "9a6fe8997eb44278b644d7ef46bfcc5f",
            "9e46e7b2d8ed46b5b4f0b5863144e4d9",
            "85fc1be89f0f410fb3c42ece5b3cf753",
            "6727b1ce5e77417a84b81300e521d5cf",
            "8e497fb09a2b46f38fbdb183f161c11b",
            "0a7b50d35a34478782587d994b92686f",
            "dc17c12200dc43d49bdd9ac553ab6aeb",
            "d5b95639da0a4fdba55db17b88b435c8",
            "e384b0da8603465da025b0c23f314c19",
            "e3dca40f33e149f8aa197b24ded041e7",
            "e692d404441847719f36d3d0309e91d2",
            "a05068727adb45318409d4f804d549fd",
            "b2938f9f76754fd1b2febeb80dc16a8c",
            "8e41f220be5f4918a3bc6cc324dbc2fd",
            "960f12ba9f9e488ea8a9f8cf2ba1dc9b",
            "49818b51fa9640a9b0ba1a7317582f09",
            "213f17650c4243f1b28f07a09cde8595",
            "62f2b37c09864529b0ac309a6a99c7fc",
            "2598da5e0ff54bac8637fa6b322b6f49",
            "5045b3a2cc8746fc959426f6c7c8d826",
            "1c45210124674119806875d1414cfefd",
            "242a8edd91f5457f9dc765c9ad9e870e",
            "d646b9f67a99498aa028208003e1ab40",
            "353630a87cda4f3daf9129d68e7f6d1c",
            "0105c914e18246e1876706e38171b0dd",
            "db4bd9bc210b4e768b45c3910162144c",
            "2bfb9df94e8b423ab0969785b947271c",
            "4212a4f27158486aa794eb4b46a12d4d",
            "bbee25cf88d043b79e99e005449acf4c",
            "0240cdee23054544a11ace7688b63b43",
            "9a315f66abe34d7a86386b95126cda7d",
            "27133a953b624b29824d8dd6378ba94f",
            "a156157825e3484484d0f745594d673a",
            "1fb61ad0ca0b42e095fb05f3509ed312",
            "09d496d04a154da89f95ea540b8cf115",
            "52aac5b11d3544638464516c6f353355",
            "117c2e8916674523a5a97268ef5a20b8",
            "89ccc3cb8fc448b79779585ec99be6f4",
            "be74d46ede584de6962fcfc9d8b23728",
            "907bc2149d3f43a3b04c2f1c4bcf9245",
            "a4a1648055b342719c1929963266b656",
            "58fba715d9eb4678965a659b3a9f4ada",
            "c0d46ba996234a28ad1b7b2047de331a",
            "883f80544a794b24a5f28b97ff6854c4",
            "2f7ce35ebda74f5da2a6b8785aac7b90",
            "eaa074ef9e7847cda5a90277b23b14c4",
            "1ec1cbddb19e44b1bf5893790c6ecae5",
            "d750d039398e42259ada18cc4cb9fdf1",
            "757547ec793648478cf460fb235d306d",
            "aaa35e20a09144b28cbe08aa889c2899",
            "7cee055816be4525a2cc7cdf391c2002",
            "0f82e6453f1d43f7b8c23f02b21dc1e4",
            "b579413909b64e1ca7a7f2431c41341b",
            "8b2960106c414ac7bffcc2df2dd291d1",
            "d423a5e64e19410fa88d3cabb4284cfa",
            "7429473f500c4495be4f72c55eb68eca",
            "1ac218f042a74c7a8810c20ffea22aed",
            "934793d5ad1d4274ac5cb99079bf7712",
            "29d3831c2f104b4c9cb038ac92998d90",
            "91bb99f4b1874158b30034371769cec6",
            "b2c53b329f694f3cb580fac8255267d3",
            "7f0c018b1d83430ebfefd96037f91d6a",
            "6925904a6e494b48a605797a79959a45",
            "b16ac046fafb4dadace6110f92298a5c",
            "a2b695322a5e4c829f84153480cb4719",
            "a9cc669710974e0aa2ad2d8f6a62de3a",
            "3d9fc12a26e742dfa7b8b2878aadf79a",
            "145e27cf13ba4573816997ce390cc208",
            "5c5bb78d039b4e0f8eee1f0c881d251b",
            "1d12488ad00b441da2d9290d1c52af10",
            "7fb1385de6244e4cb322e67d84d4b6be",
            "cbc218b7688e40daa44509d914d56f4b",
            "e9f097be855c4d7488bacebee86169d4",
            "f706c17b10974d578517be77084e55f3"
          ]
        },
        "outputId": "80ac9ec5-7f01-4bcc-8a43-f7870e0e1e18"
      },
      "source": [
        "epoch_itr = load_data_iterator(task, \"train\", config.start_epoch, config.max_tokens, config.num_workers)\n",
        "try_load_checkpoint(model, optimizer, name=config.resume)\n",
        "while epoch_itr.next_epoch_idx <= config.max_epoch:\n",
        "    # train for one epoch\n",
        "    train_one_epoch(epoch_itr, model, task, criterion, optimizer, config.accum_steps)\n",
        "    stats = validate_and_save(model, task, criterion, optimizer, epoch=epoch_itr.epoch)\n",
        "    logger.info(\"end of epoch {}\".format(epoch_itr.epoch))    \n",
        "    epoch_itr = load_data_iterator(task, \"train\", epoch_itr.next_epoch_idx, config.max_tokens, config.num_workers)"
      ],
      "execution_count": 43,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-08-03 02:35:12 | WARNING | fairseq.tasks.fairseq_task | 1 samples have invalid sizes and will be skipped, max_positions=(1024, 1024), first few sample ids=[326674]\n",
            "2021-08-03 02:35:13 | INFO | hw5.seq2seq | no checkpoints found at checkpoints/rnn/checkpoint_last.pt!\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "f857b6d1db15485096eac510203e20d4",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 1', max=791.0, style=ProgressStyle(descriptio…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:41: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 02:38:30 | INFO | hw5.seq2seq | training loss: 7.0574\n",
            "2021-08-03 02:38:30 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "9ecdb4186dd14b37a2d87f1acaa827ad",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/_tensor.py:575: UserWarning: floor_divide is deprecated, and will be removed in a future version of pytorch. It currently rounds toward 0 (like the 'trunc' function NOT 'floor'). This results in incorrect rounding for negative values.\n",
            "To keep the current behavior, use torch.div(a, b, rounding_mode='trunc'), or for actual floor division, use torch.div(a, b, rounding_mode='floor'). (Triggered internally at  /pytorch/aten/src/ATen/native/BinaryOps.cpp:467.)\n",
            "  return torch.floor_divide(self, other)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 02:38:58 | INFO | hw5.seq2seq | example source: but the point is , it doesn't feel that way .\n",
            "2021-08-03 02:38:58 | INFO | hw5.seq2seq | example hypothesis: 所以 , 這裡 , 這樣 , 這樣 。\n",
            "2021-08-03 02:38:58 | INFO | hw5.seq2seq | example reference: 但重點是 , 感覺起來卻不是那麼回事呀\n",
            "2021-08-03 02:38:58 | INFO | hw5.seq2seq | validation loss:\t5.9902\n",
            "2021-08-03 02:38:58 | INFO | hw5.seq2seq | BLEU = 0.71 14.6/2.3/0.4/0.1 (BP = 0.771 ratio = 0.793 hyp_len = 88692 ref_len = 111811)\n",
            "2021-08-03 02:38:58 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint1.pt\n",
            "2021-08-03 02:38:58 | INFO | hw5.seq2seq | end of epoch 1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "97422a5d2c104b51b78289c533aa4140",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 2', max=791.0, style=ProgressStyle(descriptio…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 02:42:17 | INFO | hw5.seq2seq | training loss: 5.5271\n",
            "2021-08-03 02:42:17 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "5efc4463901548d0ae9f3c6e390305fb",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 02:42:38 | INFO | hw5.seq2seq | example source: we can push through that .\n",
            "2021-08-03 02:42:38 | INFO | hw5.seq2seq | example hypothesis: 我們可以把這個轉換 。\n",
            "2021-08-03 02:42:38 | INFO | hw5.seq2seq | example reference: 我們可以衝破這些限制 。\n",
            "2021-08-03 02:42:38 | INFO | hw5.seq2seq | validation loss:\t5.0249\n",
            "2021-08-03 02:42:38 | INFO | hw5.seq2seq | BLEU = 7.38 41.4/15.8/6.5/2.7 (BP = 0.710 ratio = 0.745 hyp_len = 83287 ref_len = 111811)\n",
            "2021-08-03 02:42:38 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint2.pt\n",
            "2021-08-03 02:42:38 | INFO | hw5.seq2seq | end of epoch 2\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a1c8b59a443140ffaac9c55deff7fc17",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 3', max=791.0, style=ProgressStyle(descriptio…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 02:45:55 | INFO | hw5.seq2seq | training loss: 4.9204\n",
            "2021-08-03 02:45:55 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a77fdf78a3c744a9953aa32ebc56ba6e",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 02:46:20 | INFO | hw5.seq2seq | example source: thank you .\n",
            "2021-08-03 02:46:20 | INFO | hw5.seq2seq | example hypothesis: 謝謝 。\n",
            "2021-08-03 02:46:20 | INFO | hw5.seq2seq | example reference: 謝謝 。\n",
            "2021-08-03 02:46:20 | INFO | hw5.seq2seq | validation loss:\t4.5446\n",
            "2021-08-03 02:46:20 | INFO | hw5.seq2seq | BLEU = 11.90 43.4/19.0/9.1/4.6 (BP = 0.873 ratio = 0.880 hyp_len = 98448 ref_len = 111811)\n",
            "2021-08-03 02:46:20 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint3.pt\n",
            "2021-08-03 02:46:21 | INFO | hw5.seq2seq | end of epoch 3\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "1dd956561a4949629cc483805c4d30ca",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 4', max=791.0, style=ProgressStyle(descriptio…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 02:49:39 | INFO | hw5.seq2seq | training loss: 4.6210\n",
            "2021-08-03 02:49:39 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "7a3fa24175464695a5d8bc5144fcc5d8",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 02:50:06 | INFO | hw5.seq2seq | example source: those men would eventually be convicted of placing a van filled with 1 , 500 pounds of explosives into the sublevel parking lot of the world trade center's north tower , causing an explosion that killed six people and injured over 1 , 000 others .\n",
            "2021-08-03 02:50:06 | INFO | hw5.seq2seq | example hypothesis: 那些男人最終會被邀請一千五千磅的探險 , 其中五百磅的探險會被邀請在世界各地 , 到北極地區 , 到北極地區 , 到北部 , 到北部 , 被殺死了 。\n",
            "2021-08-03 02:50:06 | INFO | hw5.seq2seq | example reference: 這群男人最終被指控將滿載1500磅重的炸彈的廂式貨車停在世界貿易中心北塔的地下停車場 , 爆炸造成6人死亡 , 同時致使超過1000人受傷 。\n",
            "2021-08-03 02:50:06 | INFO | hw5.seq2seq | validation loss:\t4.3193\n",
            "2021-08-03 02:50:06 | INFO | hw5.seq2seq | BLEU = 13.69 45.7/21.2/10.7/5.7 (BP = 0.880 ratio = 0.887 hyp_len = 99125 ref_len = 111811)\n",
            "2021-08-03 02:50:06 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint4.pt\n",
            "2021-08-03 02:50:06 | INFO | hw5.seq2seq | end of epoch 4\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "9347496d609b4d22a7b13ecf0bb44320",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 5', max=791.0, style=ProgressStyle(descriptio…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 02:53:24 | INFO | hw5.seq2seq | training loss: 4.4264\n",
            "2021-08-03 02:53:24 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "ece3c14c188243baac9d1682c8005859",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 02:53:50 | INFO | hw5.seq2seq | example source: but this is really how i go about creating these photographs .\n",
            "2021-08-03 02:53:50 | INFO | hw5.seq2seq | example hypothesis: 但這真的是我如何創造這些照片 。\n",
            "2021-08-03 02:53:50 | INFO | hw5.seq2seq | example reference: 但是我真的是這樣做出這些相片的 。\n",
            "2021-08-03 02:53:50 | INFO | hw5.seq2seq | validation loss:\t4.1364\n",
            "2021-08-03 02:53:50 | INFO | hw5.seq2seq | BLEU = 16.50 48.1/23.3/12.3/6.7 (BP = 0.946 ratio = 0.947 hyp_len = 105881 ref_len = 111811)\n",
            "2021-08-03 02:53:50 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint5.pt\n",
            "2021-08-03 02:53:50 | INFO | hw5.seq2seq | end of epoch 5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e3898759a14d4e21aea94e421a4bcd29",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 6', max=791.0, style=ProgressStyle(descriptio…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 02:57:08 | INFO | hw5.seq2seq | training loss: 4.2677\n",
            "2021-08-03 02:57:08 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "afd568333e1641faa1ad279c13d1c630",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 02:57:35 | INFO | hw5.seq2seq | example source: so that is a fact .\n",
            "2021-08-03 02:57:35 | INFO | hw5.seq2seq | example hypothesis: 這是事實 。\n",
            "2021-08-03 02:57:35 | INFO | hw5.seq2seq | example reference: 這就是事實\n",
            "2021-08-03 02:57:35 | INFO | hw5.seq2seq | validation loss:\t4.0042\n",
            "2021-08-03 02:57:35 | INFO | hw5.seq2seq | BLEU = 17.97 51.5/25.9/13.9/7.9 (BP = 0.919 ratio = 0.922 hyp_len = 103104 ref_len = 111811)\n",
            "2021-08-03 02:57:35 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint6.pt\n",
            "2021-08-03 02:57:35 | INFO | hw5.seq2seq | end of epoch 6\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "864094c044844e0aa70aa6ee04d14a73",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 7', max=791.0, style=ProgressStyle(descriptio…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 03:00:53 | INFO | hw5.seq2seq | training loss: 4.1291\n",
            "2021-08-03 03:00:53 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "8d2daceaa80041629ad96f67f50ec2d7",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 03:01:19 | INFO | hw5.seq2seq | example source: thank you so much everyone .\n",
            "2021-08-03 03:01:19 | INFO | hw5.seq2seq | example hypothesis: 非常謝謝謝\n",
            "2021-08-03 03:01:19 | INFO | hw5.seq2seq | example reference: 謝謝大家 !\n",
            "2021-08-03 03:01:19 | INFO | hw5.seq2seq | validation loss:\t3.9248\n",
            "2021-08-03 03:01:19 | INFO | hw5.seq2seq | BLEU = 18.05 53.7/27.5/14.9/8.6 (BP = 0.865 ratio = 0.873 hyp_len = 97663 ref_len = 111811)\n",
            "2021-08-03 03:01:19 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint7.pt\n",
            "2021-08-03 03:01:19 | INFO | hw5.seq2seq | end of epoch 7\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "1e808665cdfa4a5998f26c3d4b51c7a5",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 8', max=791.0, style=ProgressStyle(descriptio…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 03:04:37 | INFO | hw5.seq2seq | training loss: 4.0309\n",
            "2021-08-03 03:04:37 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "da42ef922b904b76811fa04efbea3ba9",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 03:05:02 | INFO | hw5.seq2seq | example source: these differences became a source of inspired artistic celebration .\n",
            "2021-08-03 03:05:02 | INFO | hw5.seq2seq | example hypothesis: 這些差異變成了啟發藝術慶祝的源頭 。\n",
            "2021-08-03 03:05:02 | INFO | hw5.seq2seq | example reference: 這些差異變成了一個源頭 , 富有靈感之藝術讚頌的源頭 。\n",
            "2021-08-03 03:05:02 | INFO | hw5.seq2seq | validation loss:\t3.8206\n",
            "2021-08-03 03:05:02 | INFO | hw5.seq2seq | BLEU = 19.36 54.8/28.7/15.9/9.3 (BP = 0.881 ratio = 0.887 hyp_len = 99228 ref_len = 111811)\n",
            "2021-08-03 03:05:03 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint8.pt\n",
            "2021-08-03 03:05:03 | INFO | hw5.seq2seq | end of epoch 8\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3490001381064d6b94e096891c8b656d",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 9', max=791.0, style=ProgressStyle(descriptio…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 03:08:20 | INFO | hw5.seq2seq | training loss: 3.9548\n",
            "2021-08-03 03:08:20 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "ef66637c0f884002b233737b7e458feb",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 03:08:47 | INFO | hw5.seq2seq | example source: it's not .\n",
            "2021-08-03 03:08:47 | INFO | hw5.seq2seq | example hypothesis: 不是 。\n",
            "2021-08-03 03:08:47 | INFO | hw5.seq2seq | example reference: 不是\n",
            "2021-08-03 03:08:47 | INFO | hw5.seq2seq | validation loss:\t3.7896\n",
            "2021-08-03 03:08:47 | INFO | hw5.seq2seq | BLEU = 20.64 51.5/27.0/14.9/8.7 (BP = 1.000 ratio = 1.007 hyp_len = 112540 ref_len = 111811)\n",
            "2021-08-03 03:08:47 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint9.pt\n",
            "2021-08-03 03:08:47 | INFO | hw5.seq2seq | end of epoch 9\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d8ee47410b0a4556a0163485c26a1429",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 10', max=791.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 03:12:05 | INFO | hw5.seq2seq | training loss: 3.8992\n",
            "2021-08-03 03:12:05 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "582a89642fa64656b63071f49b22796d",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 03:12:28 | INFO | hw5.seq2seq | example source: so think about when you have friends , families and coworkers in california , on the west coast or in other parts of the world .\n",
            "2021-08-03 03:12:28 | INFO | hw5.seq2seq | example hypothesis: 想想看 , 當你有朋友、家庭和同事在加州 , 在西岸或其他地方 。\n",
            "2021-08-03 03:12:28 | INFO | hw5.seq2seq | example reference: 所以 , 想像當你嘗試聯繫在加州 , 在西海岸或者在世界的另一面\n",
            "2021-08-03 03:12:28 | INFO | hw5.seq2seq | validation loss:\t3.7314\n",
            "2021-08-03 03:12:28 | INFO | hw5.seq2seq | BLEU = 20.47 56.4/29.9/16.7/9.9 (BP = 0.892 ratio = 0.897 hyp_len = 100317 ref_len = 111811)\n",
            "2021-08-03 03:12:28 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint10.pt\n",
            "2021-08-03 03:12:28 | INFO | hw5.seq2seq | end of epoch 10\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "fafc50e23a88426182fd07a779935b31",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 11', max=791.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 03:15:47 | INFO | hw5.seq2seq | training loss: 3.8529\n",
            "2021-08-03 03:15:47 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b1794d354a864a7aabc11c5d2254fa13",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 03:16:11 | INFO | hw5.seq2seq | example source: and i was asked to move out because one of my roommates had shared my status with her parents .\n",
            "2021-08-03 03:16:11 | INFO | hw5.seq2seq | example hypothesis: 我被要求搬出去 , 因為我的房間之一和她的父母分享我的地位 。\n",
            "2021-08-03 03:16:11 | INFO | hw5.seq2seq | example reference: 我會被要求搬出去 , 是因為其中一名室友和她的父母談了我的狀況 。\n",
            "2021-08-03 03:16:11 | INFO | hw5.seq2seq | validation loss:\t3.7094\n",
            "2021-08-03 03:16:11 | INFO | hw5.seq2seq | BLEU = 21.25 55.6/29.7/16.7/9.8 (BP = 0.932 ratio = 0.934 hyp_len = 104452 ref_len = 111811)\n",
            "2021-08-03 03:16:11 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint11.pt\n",
            "2021-08-03 03:16:11 | INFO | hw5.seq2seq | end of epoch 11\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3d3da426d14f4da098753388612a6f4c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 12', max=791.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 03:19:28 | INFO | hw5.seq2seq | training loss: 3.8179\n",
            "2021-08-03 03:19:28 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d23b4df531884201911c24998813416c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 03:19:52 | INFO | hw5.seq2seq | example source: and i want to share with you a quick video from our facility that gives you a sense of how this looks at scale .\n",
            "2021-08-03 03:19:52 | INFO | hw5.seq2seq | example hypothesis: 我想跟各位分享一個快速的影片 , 讓我們的設備能讓你了解這有多大 。\n",
            "2021-08-03 03:19:52 | INFO | hw5.seq2seq | example reference: 我想與你們快速分享一段關於我們設備的影片可以瞭解一下它的規模\n",
            "2021-08-03 03:19:52 | INFO | hw5.seq2seq | validation loss:\t3.6643\n",
            "2021-08-03 03:19:52 | INFO | hw5.seq2seq | BLEU = 21.31 56.1/30.1/16.9/10.1 (BP = 0.921 ratio = 0.924 hyp_len = 103263 ref_len = 111811)\n",
            "2021-08-03 03:19:52 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint12.pt\n",
            "2021-08-03 03:19:52 | INFO | hw5.seq2seq | end of epoch 12\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "056fc96039164220854a7e0e9e8c2c0e",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 13', max=791.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 03:23:09 | INFO | hw5.seq2seq | training loss: 3.7861\n",
            "2021-08-03 03:23:09 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "7ba282ed8b234aa5835fc0f46b0cc9b4",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 03:23:34 | INFO | hw5.seq2seq | example source: it's a distinction perhaps we might make between two nobel laureates , richard feynman and john nash .\n",
            "2021-08-03 03:23:34 | INFO | hw5.seq2seq | example hypothesis: 或許我們可能在兩個諾貝爾雷射和約翰納許曼和約翰納許之間做分別 。\n",
            "2021-08-03 03:23:34 | INFO | hw5.seq2seq | example reference: 我們或許可以從兩位諾貝爾獎得主 , 理查費曼和約翰奈許之間看出這樣的差異 。\n",
            "2021-08-03 03:23:34 | INFO | hw5.seq2seq | validation loss:\t3.6527\n",
            "2021-08-03 03:23:34 | INFO | hw5.seq2seq | BLEU = 21.49 56.9/30.8/17.5/10.5 (BP = 0.902 ratio = 0.907 hyp_len = 101364 ref_len = 111811)\n",
            "2021-08-03 03:23:34 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint13.pt\n",
            "2021-08-03 03:23:34 | INFO | hw5.seq2seq | end of epoch 13\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "db1f4908985f428bb8d5413f876f6d93",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 14', max=791.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 03:26:52 | INFO | hw5.seq2seq | training loss: 3.7590\n",
            "2021-08-03 03:26:52 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "f0ba7e145d0d4971be09662f2e2fbf3a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 03:27:15 | INFO | hw5.seq2seq | example source: and therein lie the words of wisdom of a young girl whose brief life forever impacted mine .\n",
            "2021-08-03 03:27:15 | INFO | hw5.seq2seq | example hypothesis: 在那裡 , 年輕女孩的智慧字眼 , 會永遠影響我的人生 。\n",
            "2021-08-03 03:27:15 | INFO | hw5.seq2seq | example reference: 她說的話對我有很大啟發 。 這一個年幼女孩短暫的一生永遠影響著我 。\n",
            "2021-08-03 03:27:15 | INFO | hw5.seq2seq | validation loss:\t3.6377\n",
            "2021-08-03 03:27:15 | INFO | hw5.seq2seq | BLEU = 21.08 58.3/31.5/17.9/10.8 (BP = 0.864 ratio = 0.872 hyp_len = 97533 ref_len = 111811)\n",
            "2021-08-03 03:27:15 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint14.pt\n",
            "2021-08-03 03:27:15 | INFO | hw5.seq2seq | end of epoch 14\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "523d21c12e8144fe85d2bc5d8b8b87de",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 15', max=791.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 03:30:34 | INFO | hw5.seq2seq | training loss: 3.7397\n",
            "2021-08-03 03:30:34 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "cf92a0bfd2654884aaaf30dc7941aaf0",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 03:30:58 | INFO | hw5.seq2seq | example source: and we've started making curtains , and not only is it beautiful , but people can see status that you care about your children .\n",
            "2021-08-03 03:30:58 | INFO | hw5.seq2seq | example hypothesis: 我們開始做曲線 , 不僅是它很美麗 , 人們可以看到你關心孩子的地位 。\n",
            "2021-08-03 03:30:58 | INFO | hw5.seq2seq | example reference: 我們也開始生産窗簾不止因爲窗簾看起來漂亮 , 人們也可以看到你的地位看得出你很關心小孩\n",
            "2021-08-03 03:30:58 | INFO | hw5.seq2seq | validation loss:\t3.6186\n",
            "2021-08-03 03:30:58 | INFO | hw5.seq2seq | BLEU = 21.49 57.9/31.5/17.9/10.7 (BP = 0.883 ratio = 0.889 hyp_len = 99411 ref_len = 111811)\n",
            "2021-08-03 03:30:58 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint15.pt\n",
            "2021-08-03 03:30:58 | INFO | hw5.seq2seq | end of epoch 15\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "aa8b0134139a4514bd938aa40df3314a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 16', max=791.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 03:34:17 | INFO | hw5.seq2seq | training loss: 3.7140\n",
            "2021-08-03 03:34:17 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "daeb3902e30044cdb74f8497d5287c31",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 03:34:40 | INFO | hw5.seq2seq | example source: now , i'm not suggesting we want to raise our babies in our stomach , but i am suggesting it's possible we might want to manage gastric secretion in the gut .\n",
            "2021-08-03 03:34:40 | INFO | hw5.seq2seq | example hypothesis: 現在 , 我並不是建議我們不想在胃裡養育嬰兒 , 但我建議 , 我們可能想要管理腸道中的氣體秘密 。\n",
            "2021-08-03 03:34:40 | INFO | hw5.seq2seq | example reference: 我並不是說要把嬰兒養在胃裡而是我們可能可以了解胃在內臟裡的分泌物\n",
            "2021-08-03 03:34:40 | INFO | hw5.seq2seq | validation loss:\t3.6022\n",
            "2021-08-03 03:34:40 | INFO | hw5.seq2seq | BLEU = 22.42 56.6/30.7/17.6/10.6 (BP = 0.940 ratio = 0.942 hyp_len = 105288 ref_len = 111811)\n",
            "2021-08-03 03:34:40 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint16.pt\n",
            "2021-08-03 03:34:40 | INFO | hw5.seq2seq | end of epoch 16\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "8453f3413a55457f9ff114d37bceee39",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 17', max=791.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 03:38:00 | INFO | hw5.seq2seq | training loss: 3.6986\n",
            "2021-08-03 03:38:00 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e6e392d13e9545bf987a61139d5445b6",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 03:38:24 | INFO | hw5.seq2seq | example source: so we're hoping that's what they'll do .\n",
            "2021-08-03 03:38:24 | INFO | hw5.seq2seq | example hypothesis: 所以 , 我們希望他們會這樣做 。\n",
            "2021-08-03 03:38:24 | INFO | hw5.seq2seq | example reference: 所以 , 我們希望它們能夠幫忙 。\n",
            "2021-08-03 03:38:24 | INFO | hw5.seq2seq | validation loss:\t3.5913\n",
            "2021-08-03 03:38:24 | INFO | hw5.seq2seq | BLEU = 22.57 56.9/31.0/17.8/10.7 (BP = 0.937 ratio = 0.939 hyp_len = 104954 ref_len = 111811)\n",
            "2021-08-03 03:38:24 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint17.pt\n",
            "2021-08-03 03:38:24 | INFO | hw5.seq2seq | end of epoch 17\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "6022dba9c0fc44928150f03e4d0707f8",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 18', max=791.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 03:41:44 | INFO | hw5.seq2seq | training loss: 3.6807\n",
            "2021-08-03 03:41:44 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "1046bb04ed3540ffad21248155f92a59",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 03:42:08 | INFO | hw5.seq2seq | example source: i remember so vividly that day .\n",
            "2021-08-03 03:42:08 | INFO | hw5.seq2seq | example hypothesis: 我記得那天如此惡劣 。\n",
            "2021-08-03 03:42:08 | INFO | hw5.seq2seq | example reference: 我對那一天還記憶猶新 。\n",
            "2021-08-03 03:42:08 | INFO | hw5.seq2seq | validation loss:\t3.5788\n",
            "2021-08-03 03:42:08 | INFO | hw5.seq2seq | BLEU = 22.48 57.2/31.3/17.9/10.8 (BP = 0.927 ratio = 0.930 hyp_len = 103948 ref_len = 111811)\n",
            "2021-08-03 03:42:08 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint18.pt\n",
            "2021-08-03 03:42:08 | INFO | hw5.seq2seq | end of epoch 18\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "30bcbe1ee7f84a8ca8f9d40f77be522b",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 19', max=791.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 03:45:29 | INFO | hw5.seq2seq | training loss: 3.6668\n",
            "2021-08-03 03:45:29 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "13eba79cc9f04134801a52f71881e8b1",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 03:45:52 | INFO | hw5.seq2seq | example source: the next step is to make compliant toes , and try to add spines and claws and set it for dry adhesives .\n",
            "2021-08-03 03:45:52 | INFO | hw5.seq2seq | example hypothesis: 下一步是要製造複雜的腳趾 , 試著加上脊椎和爪子 , 並把它放到乾燥的黏土上 。\n",
            "2021-08-03 03:45:53 | INFO | hw5.seq2seq | example reference: 下一步是要製造順從聽話的腳趾頭 。 然後加上刺和爪子 , 再加上乾膠 。\n",
            "2021-08-03 03:45:53 | INFO | hw5.seq2seq | validation loss:\t3.5706\n",
            "2021-08-03 03:45:53 | INFO | hw5.seq2seq | BLEU = 22.97 57.3/31.5/18.1/11.0 (BP = 0.938 ratio = 0.940 hyp_len = 105110 ref_len = 111811)\n",
            "2021-08-03 03:45:53 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint19.pt\n",
            "2021-08-03 03:45:53 | INFO | hw5.seq2seq | end of epoch 19\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c7bca190a59d44d0b41399002a2accae",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 20', max=791.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 03:49:15 | INFO | hw5.seq2seq | training loss: 3.6526\n",
            "2021-08-03 03:49:15 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "df8ad6dae6424015baef44d5e019d13d",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 03:49:38 | INFO | hw5.seq2seq | example source: and got the cord off of the baby's neck , and a healthy screaming , kicking baby arrived , just as the dad ran in from the parking lot , \" hi , you have a son , i'm dr . darria .\n",
            "2021-08-03 03:49:38 | INFO | hw5.seq2seq | example hypothesis: 然後從嬰兒的脖子中得到玉米 , 有個健康的尖叫 , 可蘭花嬰兒抵達 , 就像爸爸從停車場跑出來的一樣 , 「 嗨 , 你有兒子 , 我是達里亞博士 。\n",
            "2021-08-03 03:49:38 | INFO | hw5.seq2seq | example reference: 我把臍帶從嬰兒的脖子繞開 , 接著 , 一個健康哭鬧的嬰兒誕生了 。 同時 , 他的父親剛從停車場趕來 。 「 你好 , 是一個男孩 。 我是達里亞醫生 。\n",
            "2021-08-03 03:49:38 | INFO | hw5.seq2seq | validation loss:\t3.5535\n",
            "2021-08-03 03:49:38 | INFO | hw5.seq2seq | BLEU = 22.87 57.5/31.6/18.2/11.1 (BP = 0.929 ratio = 0.931 hyp_len = 104106 ref_len = 111811)\n",
            "2021-08-03 03:49:38 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint20.pt\n",
            "2021-08-03 03:49:38 | INFO | hw5.seq2seq | end of epoch 20\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "7023d21c52f140a7a119edbcd783743b",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 21', max=791.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 03:53:01 | INFO | hw5.seq2seq | training loss: 3.6444\n",
            "2021-08-03 03:53:01 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c9f024091d8841d99ad01854fc1cb691",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 03:53:24 | INFO | hw5.seq2seq | example source: no , but the issue of city building in democracy is interesting because it creates chaos , right ?\n",
            "2021-08-03 03:53:24 | INFO | hw5.seq2seq | example hypothesis: 不 , 但民主的城市建築的問題很有趣 , 因為它創造混亂 , 對吧 ?\n",
            "2021-08-03 03:53:24 | INFO | hw5.seq2seq | example reference: 不 , 但民主的城市建設很有趣 , 因為會造成混亂 , 是嗎 ?\n",
            "2021-08-03 03:53:24 | INFO | hw5.seq2seq | validation loss:\t3.5414\n",
            "2021-08-03 03:53:24 | INFO | hw5.seq2seq | BLEU = 23.10 57.2/31.4/18.1/10.9 (BP = 0.947 ratio = 0.948 hyp_len = 106015 ref_len = 111811)\n",
            "2021-08-03 03:53:25 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint21.pt\n",
            "2021-08-03 03:53:25 | INFO | hw5.seq2seq | end of epoch 21\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "6c8a3569bfa2444681b7a61cf648df9a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 22', max=791.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 03:56:45 | INFO | hw5.seq2seq | training loss: 3.6276\n",
            "2021-08-03 03:56:45 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c47bad70e69b46e6a76550d86099af03",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 03:57:09 | INFO | hw5.seq2seq | example source: so i'm going to show you a demo of this .\n",
            "2021-08-03 03:57:09 | INFO | hw5.seq2seq | example hypothesis: 所以我要向你們展示一下這個 。\n",
            "2021-08-03 03:57:09 | INFO | hw5.seq2seq | example reference: 現在請各位看示範\n",
            "2021-08-03 03:57:09 | INFO | hw5.seq2seq | validation loss:\t3.5452\n",
            "2021-08-03 03:57:09 | INFO | hw5.seq2seq | BLEU = 23.00 58.0/32.0/18.5/11.2 (BP = 0.923 ratio = 0.926 hyp_len = 103539 ref_len = 111811)\n",
            "2021-08-03 03:57:10 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint22.pt\n",
            "2021-08-03 03:57:10 | INFO | hw5.seq2seq | end of epoch 22\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "44f6a968267b446d93cc9abd73e1d927",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 23', max=791.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 04:00:30 | INFO | hw5.seq2seq | training loss: 3.6195\n",
            "2021-08-03 04:00:30 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "9c2bcd73d15948f2a01be41f75da486b",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 04:00:53 | INFO | hw5.seq2seq | example source: you see , in every country where you hear about armed jihadis targeting civilians , there are also unarmed people defying those militants that you don't hear about , and those people need our support to succeed .\n",
            "2021-08-03 04:00:53 | INFO | hw5.seq2seq | example hypothesis: 你看 , 在每個國家 , 大家都聽到武裝的聖戰目標是平民 , 也有一些不確定的人來防禦那些你聽不見的軍人 , 而那些人需要我們的支持來成功 。\n",
            "2021-08-03 04:00:53 | INFO | hw5.seq2seq | example reference: 你看 , 不管在哪個國家你都會聽到武裝聖戰者針對平民百姓 , 也有很多手無寸鐵的人民公然反抗那些激進分子 , 只是你沒聽過 , 那些人需要我們的支持才能成功 。\n",
            "2021-08-03 04:00:53 | INFO | hw5.seq2seq | validation loss:\t3.5361\n",
            "2021-08-03 04:00:54 | INFO | hw5.seq2seq | BLEU = 22.93 58.4/32.3/18.7/11.3 (BP = 0.913 ratio = 0.917 hyp_len = 102498 ref_len = 111811)\n",
            "2021-08-03 04:00:54 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint23.pt\n",
            "2021-08-03 04:00:54 | INFO | hw5.seq2seq | end of epoch 23\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "fcc5d624f7e3492899da67b5728d5398",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 24', max=791.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 04:04:14 | INFO | hw5.seq2seq | training loss: 3.6075\n",
            "2021-08-03 04:04:14 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c8645e6887fa43ef83ad37dd930e4c8a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 04:04:38 | INFO | hw5.seq2seq | example source: from gezi to tahrir to elsewhere , i've seen people put their lives and livelihoods on the line .\n",
            "2021-08-03 04:04:38 | INFO | hw5.seq2seq | example hypothesis: 從gezi到tahrir到其他地方 , 我看到人們在線上生活和生活 。\n",
            "2021-08-03 04:04:38 | INFO | hw5.seq2seq | example reference: 從格濟公園 , 到解放廣場 , 再到其它地方 , 我見過許多人用他們的生命和生計做賭注 。\n",
            "2021-08-03 04:04:38 | INFO | hw5.seq2seq | validation loss:\t3.5331\n",
            "2021-08-03 04:04:38 | INFO | hw5.seq2seq | BLEU = 22.87 58.5/32.3/18.8/11.4 (BP = 0.906 ratio = 0.910 hyp_len = 101778 ref_len = 111811)\n",
            "2021-08-03 04:04:38 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint24.pt\n",
            "2021-08-03 04:04:38 | INFO | hw5.seq2seq | end of epoch 24\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "71872ce80eba4f51b0c79916b3541cc4",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 25', max=791.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 04:07:58 | INFO | hw5.seq2seq | training loss: 3.6018\n",
            "2021-08-03 04:07:58 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c7bfaca199274391866ec46a23bdedf8",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 04:08:23 | INFO | hw5.seq2seq | example source: and every project before that had been completely personal and it was a revelation when people just started commenting , started giving feedback on your code .\n",
            "2021-08-03 04:08:23 | INFO | hw5.seq2seq | example hypothesis: 之前 , 每個專案都是完全個人化的 , 而這就是一個揭露 , 當人們開始評論時 , 開始在你的程式上提供回饋 。\n",
            "2021-08-03 04:08:23 | INFO | hw5.seq2seq | example reference: 在那之前每一件計劃都是我個人的東西 , 所以當大家開始評論 , 開始對你的程式給意見時 , 真的是一種啟示 。\n",
            "2021-08-03 04:08:23 | INFO | hw5.seq2seq | validation loss:\t3.5131\n",
            "2021-08-03 04:08:23 | INFO | hw5.seq2seq | BLEU = 23.41 56.9/31.2/18.0/10.9 (BP = 0.962 ratio = 0.963 hyp_len = 107680 ref_len = 111811)\n",
            "2021-08-03 04:08:24 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint25.pt\n",
            "2021-08-03 04:08:24 | INFO | hw5.seq2seq | end of epoch 25\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "9e0fa3454b27483da48b8452bda494b1",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 26', max=791.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 04:11:44 | INFO | hw5.seq2seq | training loss: 3.5938\n",
            "2021-08-03 04:11:44 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "85fc1be89f0f410fb3c42ece5b3cf753",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 04:12:08 | INFO | hw5.seq2seq | example source: thank you so much .\n",
            "2021-08-03 04:12:08 | INFO | hw5.seq2seq | example hypothesis: 謝謝\n",
            "2021-08-03 04:12:08 | INFO | hw5.seq2seq | example reference: 感謝各位.\n",
            "2021-08-03 04:12:08 | INFO | hw5.seq2seq | validation loss:\t3.5240\n",
            "2021-08-03 04:12:08 | INFO | hw5.seq2seq | BLEU = 22.41 59.6/33.1/19.1/11.6 (BP = 0.871 ratio = 0.879 hyp_len = 98245 ref_len = 111811)\n",
            "2021-08-03 04:12:08 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint26.pt\n",
            "2021-08-03 04:12:08 | INFO | hw5.seq2seq | end of epoch 26\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e692d404441847719f36d3d0309e91d2",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 27', max=791.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 04:15:28 | INFO | hw5.seq2seq | training loss: 3.5867\n",
            "2021-08-03 04:15:28 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "2598da5e0ff54bac8637fa6b322b6f49",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 04:15:52 | INFO | hw5.seq2seq | example source: at the age of six months , virtually every one of us is able to differentiate between animate and inanimate objects .\n",
            "2021-08-03 04:15:52 | INFO | hw5.seq2seq | example hypothesis: 在六個月後 , 我們每一個人都能夠分辨動物和無生命物體 。\n",
            "2021-08-03 04:15:52 | INFO | hw5.seq2seq | example reference: 在六個月大時 , 幾乎每個人都能辨別東西是否有生命 。\n",
            "2021-08-03 04:15:52 | INFO | hw5.seq2seq | validation loss:\t3.5161\n",
            "2021-08-03 04:15:52 | INFO | hw5.seq2seq | BLEU = 23.27 58.7/32.6/18.9/11.5 (BP = 0.915 ratio = 0.919 hyp_len = 102726 ref_len = 111811)\n",
            "2021-08-03 04:15:52 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint27.pt\n",
            "2021-08-03 04:15:52 | INFO | hw5.seq2seq | end of epoch 27\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "2bfb9df94e8b423ab0969785b947271c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 28', max=791.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 04:19:13 | INFO | hw5.seq2seq | training loss: 3.5775\n",
            "2021-08-03 04:19:13 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "09d496d04a154da89f95ea540b8cf115",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 04:19:38 | INFO | hw5.seq2seq | example source: the problem is there is no such thing as a viable democracy made up of experts , zealots , politicians and spectators .\n",
            "2021-08-03 04:19:38 | INFO | hw5.seq2seq | example hypothesis: 問題是 , 身為一個可行的民主 , 由專家、零根、政治人物、尊重者組成 。\n",
            "2021-08-03 04:19:38 | INFO | hw5.seq2seq | example reference: 事實上 , 根本沒有一種民主制度是由專家 , 狂熱分子 , 政治家和旁觀者組成的\n",
            "2021-08-03 04:19:38 | INFO | hw5.seq2seq | validation loss:\t3.5004\n",
            "2021-08-03 04:19:38 | INFO | hw5.seq2seq | BLEU = 23.55 57.2/31.6/18.3/11.1 (BP = 0.958 ratio = 0.959 hyp_len = 107176 ref_len = 111811)\n",
            "2021-08-03 04:19:38 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint28.pt\n",
            "2021-08-03 04:19:38 | INFO | hw5.seq2seq | end of epoch 28\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c0d46ba996234a28ad1b7b2047de331a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 29', max=791.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 04:23:00 | INFO | hw5.seq2seq | training loss: 3.5707\n",
            "2021-08-03 04:23:00 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "7cee055816be4525a2cc7cdf391c2002",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 04:23:24 | INFO | hw5.seq2seq | example source: they're learning our culture , our language , our heritage and realizing we're just as different and just the same as each other .\n",
            "2021-08-03 04:23:24 | INFO | hw5.seq2seq | example hypothesis: 他們在學習我們的文化、我們的語言、我們的遺產 , 並意識到我們和彼此一樣不一樣 , 就像彼此一樣 。\n",
            "2021-08-03 04:23:24 | INFO | hw5.seq2seq | example reference: 他們正在習知我們的文化、語言、傳統 , 並意識到彼此不同及相同之處 。\n",
            "2021-08-03 04:23:24 | INFO | hw5.seq2seq | validation loss:\t3.4959\n",
            "2021-08-03 04:23:24 | INFO | hw5.seq2seq | BLEU = 23.49 58.0/32.1/18.6/11.3 (BP = 0.939 ratio = 0.941 hyp_len = 105187 ref_len = 111811)\n",
            "2021-08-03 04:23:24 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint29.pt\n",
            "2021-08-03 04:23:24 | INFO | hw5.seq2seq | end of epoch 29\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "29d3831c2f104b4c9cb038ac92998d90",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 30', max=791.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 04:26:45 | INFO | hw5.seq2seq | training loss: 3.5644\n",
            "2021-08-03 04:26:45 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3d9fc12a26e742dfa7b8b2878aadf79a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 04:27:09 | INFO | hw5.seq2seq | example source: but did you know that only two percent of that funding went to rebuild haitian public institutions , including its health sector ?\n",
            "2021-08-03 04:27:09 | INFO | hw5.seq2seq | example hypothesis: 但你知道只有百分之二的資金去重建海地公共機構 , 包括健康部門 ?\n",
            "2021-08-03 04:27:09 | INFO | hw5.seq2seq | example reference: 但你知不知道國際社會捐款中只有2%用去重建海地的公共體制 , 包括衞生部門 ?\n",
            "2021-08-03 04:27:09 | INFO | hw5.seq2seq | validation loss:\t3.4955\n",
            "2021-08-03 04:27:09 | INFO | hw5.seq2seq | BLEU = 23.37 58.3/32.4/18.8/11.5 (BP = 0.924 ratio = 0.927 hyp_len = 103607 ref_len = 111811)\n",
            "2021-08-03 04:27:09 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint30.pt\n",
            "2021-08-03 04:27:09 | INFO | hw5.seq2seq | end of epoch 30\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LnqAyTDDpP_P"
      },
      "source": [
        "# Submission 繳交檔案"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3O7NSDdepP_P",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e55262e7-e5a1-4209-c62a-be8dea487f08"
      },
      "source": [
        "# 把幾個 checkpoint 平均起來可以達到 ensemble 的效果\n",
        "checkdir=config.savedir\n",
        "!python ./fairseq/scripts/average_checkpoints.py \\\n",
        "--inputs {checkdir} \\\n",
        "--num-epoch-checkpoints 5 \\\n",
        "--output {checkdir}/avg_last_5_checkpoint.pt"
      ],
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Namespace(checkpoint_upper_bound=None, inputs=['./checkpoints/rnn'], num_epoch_checkpoints=5, num_update_checkpoints=None, output='./checkpoints/rnn/avg_last_5_checkpoint.pt')\n",
            "averaging checkpoints:  ['./checkpoints/rnn/checkpoint30.pt', './checkpoints/rnn/checkpoint29.pt', './checkpoints/rnn/checkpoint28.pt', './checkpoints/rnn/checkpoint27.pt', './checkpoints/rnn/checkpoint26.pt']\n",
            "Finished writing averaged checkpoint to ./checkpoints/rnn/avg_last_5_checkpoint.pt\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "syWKeRNPpP_P"
      },
      "source": [
        "## 確認生成繳交檔案的模型參數"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eP8hB67lpP_P",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 143,
          "referenced_widgets": [
            "cf5d34476217407585503b24a9511c46",
            "53d46061dccd4aebaf00ab8729c83a89",
            "c7d901c8bd154dcbabd13ec8a2ced797",
            "a10fe7be794a4e899de39ffd09f4aa26",
            "a5ecc8be77aa463f86d2caf73d7fe70f",
            "1ec8f7e5735548fabcef10480b9bbb20",
            "b2afcb3ce66c43eca41c809dadec8896",
            "3e9ffdca36174cf39be4b90397ff3661"
          ]
        },
        "outputId": "075d94e8-ae86-4e83-ef72-4c81c9556ee2"
      },
      "source": [
        "# checkpoint_last.pt : 最後一次檢驗的檔案\n",
        "# checkpoint_best.pt : 檢驗 BLEU 最高的檔案\n",
        "# avg_last_5_checkpoint.pt:　最5後個檔案平均\n",
        "try_load_checkpoint(model, name=\"avg_last_5_checkpoint.pt\")\n",
        "validate(model, task, criterion, log_to_wandb=False)\n",
        "None"
      ],
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-08-03 04:27:31 | INFO | hw5.seq2seq | loaded checkpoint checkpoints/rnn/avg_last_5_checkpoint.pt: step=unknown loss=3.4955291748046875 bleu=23.371822431008955\n",
            "2021-08-03 04:27:31 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "cf5d34476217407585503b24a9511c46",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-03 04:27:55 | INFO | hw5.seq2seq | example source: thank you .\n",
            "2021-08-03 04:27:55 | INFO | hw5.seq2seq | example hypothesis: 謝謝\n",
            "2021-08-03 04:27:55 | INFO | hw5.seq2seq | example reference: 謝謝 。\n",
            "2021-08-03 04:27:55 | INFO | hw5.seq2seq | validation loss:\t3.4834\n",
            "2021-08-03 04:27:55 | INFO | hw5.seq2seq | BLEU = 23.57 58.7/32.7/19.0/11.7 (BP = 0.923 ratio = 0.926 hyp_len = 103506 ref_len = 111811)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5P8g6cX8pP_Q"
      },
      "source": [
        "## 進行預測"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pGP6QOihpP_Q"
      },
      "source": [
        "def generate_prediction(model, task, split=\"test\", outfile=\"./prediction.txt\"):    \n",
        "    task.load_dataset(split=split, epoch=1)\n",
        "    itr = load_data_iterator(task, split, 1, config.max_tokens, config.num_workers).next_epoch_itr(shuffle=False)\n",
        "    \n",
        "    idxs = []\n",
        "    hyps = []\n",
        "\n",
        "    model.eval()\n",
        "    progress = tqdm.tqdm(itr, desc=f\"prediction\")\n",
        "    with torch.no_grad():\n",
        "        for i, sample in enumerate(progress):\n",
        "            # validation loss\n",
        "            sample = utils.move_to_cuda(sample, device=device)\n",
        "\n",
        "            # 進行推論\n",
        "            s, h, r = inference_step(sample, model)\n",
        "            \n",
        "            hyps.extend(h)\n",
        "            idxs.extend(list(sample['id']))\n",
        "            \n",
        "    # 根據 preprocess 時的順序排列\n",
        "    hyps = [x for _,x in sorted(zip(idxs,hyps))]\n",
        "    \n",
        "    with open(outfile, \"w\") as f:\n",
        "        for h in hyps:\n",
        "            f.write(h+\"\\n\")"
      ],
      "execution_count": 46,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TzagK0gGpP_Q",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 121,
          "referenced_widgets": [
            "c02582c4e9c947a58acbe53247c5f6e5",
            "a44bb9dc8e44408c9e3def209ad23b1b",
            "b3634d05f0e941ab9d9189246a2d53b6",
            "7ab252b683d84694980d8459c43aaebb",
            "7d64f7eed7894ebe8de2d6ac64b3f3e2",
            "86b984fb1a3d4f0bbbf2de552e02cb4c",
            "136b1ed7287a44caa41effe0c22f9437",
            "7d99d3cdee8747d59133b6c3cafda3d5"
          ]
        },
        "outputId": "340ab235-ce6e-4683-a123-ce505c4900cf"
      },
      "source": [
        "generate_prediction(model, task)"
      ],
      "execution_count": 47,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-08-03 04:28:27 | INFO | fairseq.data.data_utils | loaded 4,000 examples from: ./DATA/data-bin/ted2020/test.en-zh.en\n",
            "2021-08-03 04:28:27 | INFO | fairseq.data.data_utils | loaded 4,000 examples from: ./DATA/data-bin/ted2020/test.en-zh.zh\n",
            "2021-08-03 04:28:27 | INFO | fairseq.tasks.translation | ./DATA/data-bin/ted2020 test en-zh 4000 examples\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c02582c4e9c947a58acbe53247c5f6e5",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='prediction', max=17.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QH5XujTipP_Q"
      },
      "source": [
        "raise"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cml77b_GpP_R"
      },
      "source": [
        "# Back-translation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gs0rD5vMpP_R"
      },
      "source": [
        "## 訓練一個反向的翻譯模型"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VHm7-qaWpP_R"
      },
      "source": [
        "1. 將實驗的參數設定表中(config)的source_lang與target_lang互相交換\n",
        "2. 將實驗的參數設定表中(config)的savedir更改(ex. \"./checkpoints/rnn-back\")\n",
        "3. 訓練一個反向模型"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SiNt9FUzpP_R"
      },
      "source": [
        "## 利用反向模型生成額外資料"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AGlJU7zMpP_R"
      },
      "source": [
        "### 下載 monolingual data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NQnIEL7HpP_S"
      },
      "source": [
        "mono_dataset_name = 'mono'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8lS8ecZJpP_S"
      },
      "source": [
        "mono_prefix = Path(data_dir).absolute() / mono_dataset_name\n",
        "mono_prefix.mkdir(parents=True, exist_ok=True)\n",
        "\n",
        "urls = (\n",
        "    '\"https://onedrive.live.com/download?cid=3E549F3B24B238B4&resid=3E549F3B24B238B4%214986&authkey=AANUKbGfZx0kM80\"',\n",
        "# # If the above links die, use the following instead. \n",
        "#     \"https://www.csie.ntu.edu.tw/~r09922057/ML2021-hw5/ted_zh_corpus.deduped.gz\",\n",
        "# # If the above links die, use the following instead. \n",
        "#     \"https://mega.nz/#!vMNnDShR!4eHDxzlpzIpdpeQTD-htatU_C7QwcBTwGDaSeBqH534\",\n",
        ")\n",
        "file_names = (\n",
        "    'ted_zh_corpus.deduped.gz',\n",
        ")\n",
        "\n",
        "for u, f in zip(urls, file_names):\n",
        "    path = mono_prefix/f\n",
        "    if not path.exists():\n",
        "        if 'mega' in u:\n",
        "            !megadl {u} --path {path}\n",
        "        else:\n",
        "            !wget {u} -O {path}\n",
        "    else:\n",
        "        print(f'{f} is exist, skip downloading')\n",
        "    if path.suffix == \".tgz\":\n",
        "        !tar -xvf {path} -C {prefix}\n",
        "    elif path.suffix == \".zip\":\n",
        "        !unzip -o {path} -d {prefix}\n",
        "    elif path.suffix == \".gz\":\n",
        "        !gzip -fkd {path}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OVkHLwpYpP_S"
      },
      "source": [
        "### TODO: 清理資料集\n",
        "\n",
        "1. 將太長、太短的句子移除\n",
        "2. 統一標點符號\n",
        "\n",
        "hint: 可以使用clean_s()來協助"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R7jxsGbUpP_S"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VIDJl34epP_T"
      },
      "source": [
        "### TODO: Subword Units\n",
        "\n",
        "用反向模型的 spm model 將資料切成 subword units\n",
        "\n",
        "hint: spm model 的路徑為 DATA/raw-data/\\[dataset\\]/spm\\[vocab_num\\].model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nYUvoNqApP_T"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zPsxOqoJpP_T"
      },
      "source": [
        "### Binarize\n",
        "\n",
        "使用fairseq將資料轉為binary"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8EZPJXYMpP_T"
      },
      "source": [
        "binpath = Path('./DATA/data-bin', mono_dataset_name)\n",
        "src_dict_file = './DATA/data-bin/ted2020/dict.en.txt'\n",
        "tgt_dict_file = src_dict_file\n",
        "monopref = str(mono_prefix/\"mono.tok\") # whatever filepath you get after applying subword tokenization\n",
        "if binpath.exists():\n",
        "    print(binpath, \"exists, will not overwrite!\")\n",
        "else:\n",
        "    !python -m fairseq_cli.preprocess\\\n",
        "        --source-lang 'zh'\\\n",
        "        --target-lang 'en'\\\n",
        "        --trainpref {monopref}\\\n",
        "        --destdir {binpath}\\\n",
        "        --srcdict {src_dict_file}\\\n",
        "        --tgtdict {tgt_dict_file}\\\n",
        "        --workers 2"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bFaBn9gypP_T"
      },
      "source": [
        "### TODO: 生成反向翻譯資料\n",
        "\n",
        "將 binarized data 加入原本的資料夾中並用一個 split_name 取名\n",
        "\n",
        "ex. ./DATA/data-bin/ted2020/\\[split_name\\].zh-en.\\[\"en\", \"zh\"\\].\\[\"bin\", \"idx\"\\]\n",
        "\n",
        "便可以使用 generate_prediction(model, task, split=\"split_name\")來產生翻譯資料"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ePmzhiQlpP_U"
      },
      "source": [
        "# 將 binarized data 加入原本的資料夾中並用一個 split_name 取名\n",
        "# ex. ./DATA/data-bin/ted2020/\\[split_name\\].zh-en.\\[\"en\", \"zh\"\\].\\[\"bin\", \"idx\"\\]\n",
        "!cp ./DATA/data-bin/mono/train.zh-en.zh.bin ./DATA/data-bin/ted2020/mono.zh-en.zh.bin\n",
        "!cp ./DATA/data-bin/mono/train.zh-en.zh.idx ./DATA/data-bin/ted2020/mono.zh-en.zh.idx\n",
        "!cp ./DATA/data-bin/mono/train.zh-en.en.bin ./DATA/data-bin/ted2020/mono.zh-en.en.bin\n",
        "!cp ./DATA/data-bin/mono/train.zh-en.en.idx ./DATA/data-bin/ted2020/mono.zh-en.en.idx"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NoQeVtT6pP_U"
      },
      "source": [
        "# hint: 用反向模型在 split='mono' 上進行預測，生成 prediction_file\n",
        "# generate_prediction( ... ,split=... ,outfile=... )"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "k7E4Tq9NpP_U"
      },
      "source": [
        "### TODO: 產生新的dataset\n",
        "\n",
        "1. 將翻譯出來的資料與原先的訓練資料結合\n",
        "2. 使用之前的spm model切出成Subword Units\n",
        "3. 重新使用fairseq將資料轉為binary"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3CP9u-KCpP_U"
      },
      "source": [
        "# 合併剛剛生成的 prediction_file (.en) 以及中文 mono.zh (.zh)\n",
        "# \n",
        "# hint: 在此用剛剛的 spm model 對 prediction_file 進行切斷詞\n",
        "# spm_model.encode(line, out_type=str)\n",
        "# output: ./DATA/rawdata/mono/mono.tok.en & mono.tok.zh\n",
        "#\n",
        "# hint: 在此用 fairseq 把這些檔案再 binarize\n",
        "# binpath = Path('./DATA/data-bin/synthetic')\n",
        "# src_dict_file = './DATA/data-bin/ted2020/dict.en.txt'\n",
        "# tgt_dict_file = src_dict_file\n",
        "# monopref = ./DATA/rawdata/mono/mono.tok # or whatever path after applying subword tokenization, w/o the suffix (.zh/.en)\n",
        "# if binpath.exists():\n",
        "#     print(binpath, \"exists, will not overwrite!\")\n",
        "# else:\n",
        "#     !python -m fairseq_cli.preprocess\\\n",
        "#         --source-lang 'zh'\\\n",
        "#         --target-lang 'en'\\\n",
        "#         --trainpref {monopref}\\\n",
        "#         --destdir {binpath}\\\n",
        "#         --srcdict {src_dict_file}\\\n",
        "#         --tgtdict {tgt_dict_file}\\\n",
        "#         --workers 2"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F-GTT4sypP_V"
      },
      "source": [
        "# 這裡用剛剛準備的檔案合併原先 ted2020 來生成最終 back-translation 的資料\n",
        "!cp -r ./DATA/data-bin/ted2020/ ./DATA/data-bin/ted2020_with_mono/\n",
        "\n",
        "!cp ./DATA/data-bin/synthetic/train.zh-en.zh.bin ./DATA/data-bin/ted2020_with_mono/train1.en-zh.zh.bin\n",
        "!cp ./DATA/data-bin/synthetic/train.zh-en.zh.idx ./DATA/data-bin/ted2020_with_mono/train1.en-zh.zh.idx\n",
        "!cp ./DATA/data-bin/synthetic/train.zh-en.en.bin ./DATA/data-bin/ted2020_with_mono/train1.en-zh.en.bin\n",
        "!cp ./DATA/data-bin/synthetic/train.zh-en.en.idx ./DATA/data-bin/ted2020_with_mono/train1.en-zh.en.idx"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BIJ_lVn4pP_V"
      },
      "source": [
        "### TODO: 重新訓練\n",
        "\n",
        "當已經產生新的資料集\n",
        "\n",
        "1. 將實驗的參數設定表(config)中的datadir改為新的資料集(\"./DATA/data-bin/ted2020_with_mono\")\n",
        "2. 將實驗的參數設定表(config)中的source_lang與target_lang設定還原(\"en\", \"zh\")\n",
        "3. 將實驗的參數設定表(config)中的savedir更改(ex. \"./checkpoints/rnn-bt\")\n",
        "4. 重新訓練"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aT-NU6mypP_V"
      },
      "source": [
        "# References"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vAvFeEQ2pP_W"
      },
      "source": [
        "1. <a name=ott2019fairseq></a>Ott, M., Edunov, S., Baevski, A., Fan, A., Gross, S., Ng, N., ... & Auli, M. (2019, June). fairseq: A Fast, Extensible Toolkit for Sequence Modeling. In Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics (Demonstrations) (pp. 48-53).\n",
        "2. <a name=vaswani2017></a>Vaswani, A., Shazeer, N., Parmar, N., Uszkoreit, J., Jones, L., Gomez, A. N., ... & Polosukhin, I. (2017, December). Attention is all you need. In Proceedings of the 31st International Conference on Neural Information Processing Systems (pp. 6000-6010).\n",
        "3. <a name=reimers-2020-multilingual-sentence-bert></a>Reimers, N., & Gurevych, I. (2020, November). Making Monolingual Sentence Embeddings Multilingual Using Knowledge Distillation. In Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP) (pp. 4512-4525).\n",
        "4. <a name=tiedemann2012parallel></a>Tiedemann, J. (2012, May). Parallel Data, Tools and Interfaces in OPUS. In Lrec (Vol. 2012, pp. 2214-2218).\n",
        "5. <a name=kudo-richardson-2018-sentencepiece></a>Kudo, T., & Richardson, J. (2018, November). SentencePiece: A simple and language independent subword tokenizer and detokenizer for Neural Text Processing. In Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing: System Demonstrations (pp. 66-71).\n",
        "6. <a name=sennrich-etal-2016-improving></a>Sennrich, R., Haddow, B., & Birch, A. (2016, August). Improving Neural Machine Translation Models with Monolingual Data. In Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers) (pp. 86-96).\n",
        "7. <a name=edunov-etal-2018-understanding></a>Edunov, S., Ott, M., Auli, M., & Grangier, D. (2018). Understanding Back-Translation at Scale. In Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing (pp. 489-500).\n",
        "8. https://github.com/ajinkyakulkarni14/TED-Multilingual-Parallel-Corpus\n",
        "9. https://ithelp.ithome.com.tw/articles/10233122\n",
        "10. https://nlp.seas.harvard.edu/2018/04/03/attention.html"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dh7TgKRupP_W"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}